feat: refactor internal event submission pipeline and test suites (#151)

* intitial refactor:

- Add Context-based graceful shutdown and drain phase for enqueued requests
- Use buffered channels vs. mutex for concurrency mgmt.
- Use atomic.Pointer vs. locks for FF mgmt
- Use Callback ifc vs. lossy internal tracking for unit test result validation
- Add new APIs mirroring current that accept caller's Context to control
  operation timeouts

* expose batch submission timeout with default equiv to old hardcoded timeout

* restructure event submission and batching pipeline

* overhaul tests; implement new stress tests and address coverage gaps

* parallelize tests where safe to do so (3x CI time savings!)

* introduce faster pure Go JSON seralization library

* more refactors: unit tests & benchmarks

* DRY up shutdown req drain and prepareForSend serializer

* use go-json in CLI utils too; remove redundant helpers

* move error type to proper pkg

* DRY up loop() and fix deadlock in old test case

* normalize CI and Makefile test and benchmark target behavviors

* more realistic feature flags benchmark

* replace leaky time.After pattern with idiomatic safer Timers

* internal queue size minimums decoupled from batch and worker configs

* remove dead comments

* fix test bug

* update default http.Transport cfg to avoid conn pool as bottleneck; update batch enqueue limiting strategy and tests/benches

* adjust stress test defaults to match prod

* update flaky tests and rm stale file

* make graceful shutdown timeout opt-in for now

* fix tests that reverted to hold helper utils during rebase

* fix flaky test due to overhead from -race detector now default in test flags

* add more graceful shutdown test cases
This commit is contained in:
Eli Reisman
2026-01-13 12:22:44 -08:00
committed by GitHub
parent 5ce035676b
commit 8a6e9ae51e
38 changed files with 5675 additions and 2198 deletions
+1 -1
View File
@@ -21,4 +21,4 @@ jobs:
run: go build . run: go build .
- name: Test - name: Test
run: go test -timeout 30s -v ./... run: go test -v -race -count=1 -timeout=5m -coverprofile=coverage.out ./...
+10 -2
View File
@@ -10,13 +10,21 @@ vet:
test: vet test: vet
@mkdir -p ${CIRCLE_ARTIFACTS} @mkdir -p ${CIRCLE_ARTIFACTS}
@go test -race -coverprofile=${CIRCLE_ARTIFACTS}/cover.out . @go test -v -race -count=1 -timeout=5m -coverprofile=${CIRCLE_ARTIFACTS}/cover.out ./...
@go tool cover -func ${CIRCLE_ARTIFACTS}/cover.out -o ${CIRCLE_ARTIFACTS}/cover.txt @go tool cover -func ${CIRCLE_ARTIFACTS}/cover.out -o ${CIRCLE_ARTIFACTS}/cover.txt
@go tool cover -html ${CIRCLE_ARTIFACTS}/cover.out -o ${CIRCLE_ARTIFACTS}/cover.html @go tool cover -html ${CIRCLE_ARTIFACTS}/cover.out -o ${CIRCLE_ARTIFACTS}/cover.html
# Run benchmarks (no race - it skews performance measurements)
bench: vet
@go test -v -bench=. -benchmem -run=^$$ -timeout=30m .
# Quick benchmark run (single iteration, no memory stats)
bench-quick: vet
@go test -bench=. -run=^$$ .
build: test build: test
@go build ./... @go build ./...
ci: dependencies test ci: dependencies test
.PHONY: dependencies vet test ci .PHONY: dependencies vet test bench bench-quick ci
+4 -1
View File
@@ -1,6 +1,8 @@
package posthog package posthog
import "time" import (
"time"
)
var _ Message = (*Alias)(nil) var _ Message = (*Alias)(nil)
@@ -79,3 +81,4 @@ func (msg Alias) APIfy() APIMessage {
return apified return apified
} }
+3 -1
View File
@@ -66,10 +66,12 @@ func (b *Backoff) NewTicker() *Ticker {
go func() { go func() {
for i := 0; ; i++ { for i := 0; ; i++ {
timer := time.NewTimer(b.Duration(i))
select { select {
case t := <-time.After(b.Duration(i)): case t := <-timer.C:
c <- t c <- t
case <-ticker.done: case <-ticker.done:
timer.Stop()
close(c) close(c)
return return
} }
+539
View File
@@ -0,0 +1,539 @@
package posthog
import (
"context"
"net/http"
json "github.com/goccy/go-json"
"net/http/httptest"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/require"
)
// TestBatching_SmallEventsBatchTogether verifies that small events are batched together
func TestBatching_SmallEventsBatchTogether(t *testing.T) {
t.Parallel()
var batchCount atomic.Int64
var totalMessages atomic.Int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var b batch
json.NewDecoder(r.Body).Decode(&b)
batchCount.Add(1)
totalMessages.Add(int64(len(b.Messages)))
w.WriteHeader(200)
}))
defer server.Close()
// Small events (~100 props, ~5KB each)
// With 500KB batch limit, should fit ~100 events per batch
client, err := NewWithConfig("test-key", Config{
Endpoint: server.URL,
BatchSize: 250, // Use default
Interval: 5 * time.Second, // Long interval - rely on Close() to flush
})
require.NoError(t, err)
// Send 50 small events - should all fit in one batch
pool := NewEventPoolWithCardinality(50, CardinalityLow)
for i := 0; i < 50; i++ {
err := client.Enqueue(pool.Next())
require.NoError(t, err)
}
client.Close()
require.Equal(t, int64(50), totalMessages.Load(), "All 50 events should be delivered")
// Small events should batch together efficiently
require.LessOrEqual(t, batchCount.Load(), int64(3), "50 small events should fit in 3 or fewer batches")
}
// TestBatching_LargeEventsTriggerFlush verifies that large events trigger batch flushes
func TestBatching_LargeEventsTriggerFlush(t *testing.T) {
t.Parallel()
var batchCount atomic.Int64
var batchSizes []int
var mu sync.Mutex
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var b batch
json.NewDecoder(r.Body).Decode(&b)
batchCount.Add(1)
mu.Lock()
batchSizes = append(batchSizes, len(b.Messages))
mu.Unlock()
w.WriteHeader(200)
}))
defer server.Close()
// Medium-high cardinality events (~2000 props, ~100KB each)
// With 500KB batch limit, should fit ~5 events per batch
client, err := NewWithConfig("test-key", Config{
Endpoint: server.URL,
BatchSize: 250, // Use default - byte limit should trigger before count
Interval: 50 * time.Millisecond,
})
require.NoError(t, err)
// Send 10 medium-high cardinality events
pool := NewEventPoolWithCardinality(10, CardinalityMedium)
for i := 0; i < 10; i++ {
err := client.Enqueue(pool.Next())
require.NoError(t, err)
}
client.Close()
// Medium cardinality events should trigger multiple batches
require.GreaterOrEqual(t, batchCount.Load(), int64(1), "Should have at least 1 batch")
t.Logf("Batch count: %d, sizes: %v", batchCount.Load(), batchSizes)
}
// TestBatching_OversizedEventRejected verifies that events >500KB are rejected
func TestBatching_OversizedEventRejected(t *testing.T) {
t.Parallel()
var received atomic.Int64
var failureCount atomic.Int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var b batch
json.NewDecoder(r.Body).Decode(&b)
received.Add(int64(len(b.Messages)))
w.WriteHeader(200)
}))
defer server.Close()
callback := &testCallbackCounter{
onFailure: func() { failureCount.Add(1) },
}
client, err := NewWithConfig("test-key", Config{
Endpoint: server.URL,
Callback: callback,
})
require.NoError(t, err)
// Create an event with MANY properties to exceed 500KB
// Each property is ~50 bytes, need ~10000 properties to hit 500KB
oversizedProps := make(Properties, 15000)
for i := 0; i < 15000; i++ {
oversizedProps[generateDistinctId(i)] = generateDistinctId(i + 100000)
}
err = client.Enqueue(Capture{
DistinctId: "user_1",
Event: "oversized_event",
Properties: oversizedProps,
})
require.NoError(t, err) // Enqueue itself doesn't fail
client.Close()
// The oversized event should be rejected via callback
require.Equal(t, int64(0), received.Load(), "Oversized event should not be delivered")
require.Equal(t, int64(1), failureCount.Load(), "Should have 1 failure callback for oversized event")
}
// TestBatching_MixedCardinalityBatching verifies correct batching with mixed event sizes
func TestBatching_MixedCardinalityBatching(t *testing.T) {
t.Parallel()
var batchCount atomic.Int64
var totalMessages atomic.Int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var b batch
json.NewDecoder(r.Body).Decode(&b)
batchCount.Add(1)
totalMessages.Add(int64(len(b.Messages)))
w.WriteHeader(200)
}))
defer server.Close()
client, err := NewWithConfig("test-key", Config{
Endpoint: server.URL,
BatchSize: 250,
Interval: 50 * time.Millisecond,
})
require.NoError(t, err)
// Send mix of small and medium events
smallPool := NewEventPoolWithCardinality(30, CardinalityLow)
mediumPool := NewEventPoolWithCardinality(20, CardinalityMedium)
// Interleave small and medium events
for i := 0; i < 30; i++ {
err := client.Enqueue(smallPool.Next())
require.NoError(t, err)
if i < 20 {
err := client.Enqueue(mediumPool.Next())
require.NoError(t, err)
}
}
client.Close()
require.Equal(t, int64(50), totalMessages.Load(), "All 50 events should be delivered")
t.Logf("Batch count for mixed cardinality: %d", batchCount.Load())
}
// TestBatching_BatchCountLimit verifies BatchSize config is respected
func TestBatching_BatchCountLimit(t *testing.T) {
t.Parallel()
var batchSizes []int
var mu sync.Mutex
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var b batch
json.NewDecoder(r.Body).Decode(&b)
mu.Lock()
batchSizes = append(batchSizes, len(b.Messages))
mu.Unlock()
w.WriteHeader(200)
}))
defer server.Close()
batchSize := 10
client, err := NewWithConfig("test-key", Config{
Endpoint: server.URL,
BatchSize: batchSize,
Interval: 10 * time.Millisecond, // Short interval so batch count limit triggers
})
require.NoError(t, err)
// Send 25 small events with BatchSize=10 and short interval
// The batch count limit should trigger before interval flush
pool := NewEventPoolWithCardinality(25, CardinalityLow)
for i := 0; i < 25; i++ {
err := client.Enqueue(pool.Next())
require.NoError(t, err)
}
// Wait for batches to be flushed before closing
time.Sleep(100 * time.Millisecond)
client.Close()
// Check that no batch exceeds BatchSize
for i, size := range batchSizes {
require.LessOrEqual(t, size, batchSize, "Batch %d has size %d which exceeds BatchSize %d", i, size, batchSize)
}
t.Logf("Batch sizes: %v", batchSizes)
}
// testCallbackCounter is a simple callback for counting successes and failures
type testCallbackCounter struct {
onSuccess func()
onFailure func()
}
func (c *testCallbackCounter) Success(msg APIMessage) {
if c.onSuccess != nil {
c.onSuccess()
}
}
func (c *testCallbackCounter) Failure(msg APIMessage, err error) {
if c.onFailure != nil {
c.onFailure()
}
}
// TestConfigBatchSubmitTimeout verifies the default and custom BatchSubmitTimeout config
func TestConfigBatchSubmitTimeout(t *testing.T) {
t.Parallel()
// Test default value
cfg := makeConfig(Config{})
require.Equal(t, DefaultBatchSubmitTimeout, cfg.BatchSubmitTimeout, "default BatchSubmitTimeout should be %v", DefaultBatchSubmitTimeout)
// Test custom value
customTimeout := 200 * time.Millisecond
cfg = makeConfig(Config{BatchSubmitTimeout: customTimeout})
require.Equal(t, customTimeout, cfg.BatchSubmitTimeout, "custom BatchSubmitTimeout should be preserved")
// Test negative value (non-blocking mode)
cfg = makeConfig(Config{BatchSubmitTimeout: -1})
require.Equal(t, time.Duration(-1), cfg.BatchSubmitTimeout, "negative BatchSubmitTimeout should be preserved")
}
// TestBatchSubmitTimeout_WaitsForWorkers verifies that batch submission waits
// when queue is full, giving workers time to complete during latency spikes.
// Note: params are tuned for reliability under -race (adds ~10x overhead).
func TestBatchSubmitTimeout_WaitsForWorkers(t *testing.T) {
t.Parallel()
var mu sync.Mutex
var batchCount int
// Create a handler with moderate latency
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(20 * time.Millisecond)
mu.Lock()
batchCount++
mu.Unlock()
w.WriteHeader(200)
}))
defer server.Close()
// Config: queue buffer = MaxEnqueuedRequests = 10
// With 20 events, queue will fill and submissions must wait for goroutines
client, err := NewWithConfig("test-key", Config{
Endpoint: server.URL,
BatchSize: 1, // 1 event per batch
Interval: 1 * time.Millisecond, // Flush immediately
MaxEnqueuedRequests: 10, // Queue buffer = 10
BatchSubmitTimeout: 500 * time.Millisecond, // Ample time for race mode
})
require.NoError(t, err)
// Send 20 events - will exceed queue buffer (10), so some must wait
for i := 0; i < 20; i++ {
client.Enqueue(Capture{
DistinctId: "test-user",
Event: "test-event",
})
}
client.Close()
mu.Lock()
finalBatchCount := batchCount
mu.Unlock()
// With 500ms timeout and 20ms latency, 10 workers can process ~250 batches
// All 20 events should be delivered
require.GreaterOrEqual(t, finalBatchCount, 18, "With BatchSubmitTimeout, nearly all events should be delivered")
}
// TestBatchSubmitTimeout_NonBlocking verifies that negative timeout gives non-blocking behavior
func TestBatchSubmitTimeout_NonBlocking(t *testing.T) {
t.Parallel()
var mu sync.Mutex
var successCount int
var failureCount int
// Create a very slow handler to saturate workers
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(500 * time.Millisecond) // Very slow backend
w.WriteHeader(200)
}))
defer server.Close()
callback := &testCallbackCounter{
onSuccess: func() {
mu.Lock()
successCount++
mu.Unlock()
},
onFailure: func() {
mu.Lock()
failureCount++
mu.Unlock()
},
}
// Use non-blocking mode with negative timeout
client, err := NewWithConfig("test-key", Config{
Endpoint: server.URL,
BatchSize: 1, // 1 event per batch
Interval: 1 * time.Millisecond, // Flush immediately
MaxEnqueuedRequests: 1, // Only 1 batch can be queued
BatchSubmitTimeout: -1, // Non-blocking (immediate drop)
Callback: callback,
})
require.NoError(t, err)
// Blast events as fast as possible - no sleep between enqueues.
// This ensures the channel fills faster than processBatch goroutines can drain it,
// causing the non-blocking send to drop events when the queue is full.
for i := 0; i < 100; i++ {
client.Enqueue(Capture{
DistinctId: "test-user",
Event: "test-event",
})
}
client.Close()
mu.Lock()
finalSuccess := successCount
finalFailure := failureCount
mu.Unlock()
t.Logf("Non-blocking mode: %d succeeded, %d failed", finalSuccess, finalFailure)
// In non-blocking mode with slow backend, some events should be dropped
require.Greater(t, finalFailure, 0, "In non-blocking mode with slow backend, some events should be dropped")
}
// TestShutdownTimeout_DefaultWaitsForCompletion verifies that with default config
// (ShutdownTimeout=0), Close() waits indefinitely for in-flight batches to complete.
func TestShutdownTimeout_DefaultWaitsForCompletion(t *testing.T) {
t.Parallel()
var received atomic.Int64
// Create a slow server (200ms per request)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(200 * time.Millisecond)
var b batch
json.NewDecoder(r.Body).Decode(&b)
received.Add(int64(len(b.Messages)))
w.WriteHeader(200)
}))
defer server.Close()
// Default config - ShutdownTimeout is zero (wait indefinitely)
client, err := NewWithConfig("test-key", Config{
Endpoint: server.URL,
BatchSize: 5,
Interval: 10 * time.Millisecond,
// ShutdownTimeout is intentionally not set (zero value = wait indefinitely)
})
require.NoError(t, err)
// Send events
for i := 0; i < 10; i++ {
err := client.Enqueue(Capture{
DistinctId: "test-user",
Event: "test-event",
})
require.NoError(t, err)
}
// Close should wait for all events to be delivered despite slow server
start := time.Now()
err = client.Close()
elapsed := time.Since(start)
require.NoError(t, err, "Close() should succeed without error when waiting indefinitely")
require.Equal(t, int64(10), received.Load(), "All events should be delivered")
require.GreaterOrEqual(t, elapsed, 200*time.Millisecond, "Close() should have waited for slow server")
t.Logf("Close() waited %v for slow server to complete", elapsed)
}
// TestShutdownTimeout_AbortsAfterTimeout verifies that when ShutdownTimeout is set,
// Close() aborts in-flight requests after the timeout and returns an error.
func TestShutdownTimeout_AbortsAfterTimeout(t *testing.T) {
t.Parallel()
var received atomic.Int64
var mu sync.Mutex
var failureCount int
// Create a very slow server (2s per request) - will exceed our timeout
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(2 * time.Second)
var b batch
json.NewDecoder(r.Body).Decode(&b)
received.Add(int64(len(b.Messages)))
w.WriteHeader(200)
}))
defer server.Close()
callback := &testCallbackCounter{
onFailure: func() {
mu.Lock()
failureCount++
mu.Unlock()
},
}
// Set a short shutdown timeout (100ms)
client, err := NewWithConfig("test-key", Config{
Endpoint: server.URL,
BatchSize: 5,
Interval: 10 * time.Millisecond,
ShutdownTimeout: 100 * time.Millisecond, // Short timeout - will abort
Callback: callback,
})
require.NoError(t, err)
// Send events
for i := 0; i < 10; i++ {
err := client.Enqueue(Capture{
DistinctId: "test-user",
Event: "test-event",
})
require.NoError(t, err)
}
// Close should abort after timeout
start := time.Now()
err = client.Close()
elapsed := time.Since(start)
// Should return a timeout error
require.Error(t, err, "Close() should return error when timeout is exceeded")
require.Contains(t, err.Error(), "shutdown timeout", "Error should mention shutdown timeout")
// Should have aborted relatively quickly (not waited 2s for server)
require.Less(t, elapsed, 500*time.Millisecond, "Close() should abort quickly, not wait for slow server")
// Some events may have been dropped
mu.Lock()
failures := failureCount
mu.Unlock()
t.Logf("Close() aborted after %v, received=%d, failures=%d", elapsed, received.Load(), failures)
}
// TestCloseWithContext_RespectsDeadline verifies that CloseWithContext honors
// the provided context's deadline for shutdown.
func TestCloseWithContext_RespectsDeadline(t *testing.T) {
t.Parallel()
var received atomic.Int64
// Create a slow server
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(2 * time.Second)
var b batch
json.NewDecoder(r.Body).Decode(&b)
received.Add(int64(len(b.Messages)))
w.WriteHeader(200)
}))
defer server.Close()
// No ShutdownTimeout configured - will use context deadline instead
client, err := NewWithConfig("test-key", Config{
Endpoint: server.URL,
BatchSize: 5,
Interval: 10 * time.Millisecond,
})
require.NoError(t, err)
// Send events
for i := 0; i < 10; i++ {
err := client.Enqueue(Capture{
DistinctId: "test-user",
Event: "test-event",
})
require.NoError(t, err)
}
// Use CloseWithContext with a short deadline
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
defer cancel()
start := time.Now()
err = client.CloseWithContext(ctx)
elapsed := time.Since(start)
// Should return a timeout error
require.Error(t, err, "CloseWithContext should return error when context deadline exceeded")
require.Contains(t, err.Error(), "shutdown timeout", "Error should mention shutdown timeout")
// Should have aborted at context deadline
require.Less(t, elapsed, 500*time.Millisecond, "CloseWithContext should respect context deadline")
t.Logf("CloseWithContext aborted after %v", elapsed)
}
+446
View File
@@ -0,0 +1,446 @@
package posthog
import (
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
)
// BenchmarkCallback is a lightweight callback for benchmarks
// Only tracks failures to avoid overhead on the success path
type BenchmarkCallback struct {
failureCount atomic.Int64
}
func (c *BenchmarkCallback) Success(msg APIMessage) {
// No-op - tracking successes would add overhead to every message
}
func (c *BenchmarkCallback) Failure(msg APIMessage, err error) {
c.failureCount.Add(1)
}
func (c *BenchmarkCallback) FailureCount() int64 {
return c.failureCount.Load()
}
// BenchmarkConcurrentEnqueue measures throughput at different concurrency levels
// Tests: 1, 10, 100, 500, 1000 concurrent goroutines
// Uses NoOpTransport to measure pure enqueue throughput without HTTP overhead
// Note: Under extreme load, some backpressure (delivery failures) is expected
func BenchmarkConcurrentEnqueue(b *testing.B) {
concurrencyLevels := []int{1, 10, 100, 500, 1000}
for _, concurrency := range concurrencyLevels {
b.Run(fmt.Sprintf("goroutines_%d", concurrency), func(b *testing.B) {
// Pre-generate events BEFORE benchmark
pool := NewEventPool(b.N + 1000) // Extra events to avoid index issues
callback := &BenchmarkCallback{}
var enqueueErrors atomic.Int64
client, _ := NewWithConfig("test-key", Config{
Transport: NoOpTransport(),
Callback: callback,
// Uses production defaults for BatchSize, MaxEnqueuedRequests
})
defer client.Close()
b.ResetTimer()
b.SetParallelism(concurrency)
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
if err := client.Enqueue(pool.Next()); err != nil {
enqueueErrors.Add(1)
}
}
})
b.StopTimer()
// Enqueue errors indicate channel full - fail immediately
if enqueueErrors.Load() > 0 {
b.Fatalf("Benchmark invalid: %d enqueue errors (msgs channel full)", enqueueErrors.Load())
}
// Delivery failures indicate batches channel backpressure - report but don't fail
if failures := callback.FailureCount(); failures > 0 {
b.Logf("Note: %d delivery failures (backpressure at %d goroutines)", failures, concurrency)
}
})
}
}
// BenchmarkConcurrentEnqueueWithCardinality measures throughput with different property cardinalities
// Uses NoOpTransport to measure pure enqueue throughput without HTTP overhead
// Note: Under extreme load, some backpressure (delivery failures) is expected
func BenchmarkConcurrentEnqueueWithCardinality(b *testing.B) {
cardinalities := []struct {
name string
cardinality PropertyCardinality
}{
{"low_10-100_props", CardinalityLow},
{"medium_500-2000_props", CardinalityMedium},
{"high_2000-10000_props", CardinalityHigh},
}
concurrency := 100
for _, tc := range cardinalities {
b.Run(tc.name, func(b *testing.B) {
// Use cardinality-appropriate pool size with pre-cloned unique maps
pool := NewEventPoolWithDefaultSize(tc.cardinality)
callback := &BenchmarkCallback{}
var enqueueErrors atomic.Int64
client, _ := NewWithConfig("test-key", Config{
Transport: NoOpTransport(),
Callback: callback,
// Uses production defaults for BatchSize, MaxEnqueuedRequests
})
defer client.Close()
b.ResetTimer()
b.SetParallelism(concurrency)
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
if err := client.Enqueue(pool.Next()); err != nil {
enqueueErrors.Add(1)
}
}
})
b.StopTimer()
// Enqueue errors indicate channel full - fail immediately
if enqueueErrors.Load() > 0 {
b.Fatalf("Benchmark invalid: %d enqueue errors (msgs channel full)", enqueueErrors.Load())
}
// Delivery failures indicate batches channel backpressure - report but don't fail
if failures := callback.FailureCount(); failures > 0 {
b.Logf("Note: %d delivery failures (backpressure at %d goroutines)", failures, concurrency)
}
})
}
}
// BenchmarkEnqueueThroughput measures raw enqueue speed (no HTTP)
// Uses production defaults for BatchSize, MaxEnqueuedRequests
func BenchmarkEnqueueThroughput(b *testing.B) {
pool := NewEventPool(b.N + 1000)
callback := &BenchmarkCallback{}
var enqueueErrors atomic.Int64
// Use a transport that returns immediately (no actual HTTP)
client, _ := NewWithConfig("test-key", Config{
Transport: NoOpTransport(),
Callback: callback,
// Uses production defaults for BatchSize, MaxEnqueuedRequests
})
defer client.Close()
b.ResetTimer()
for i := 0; i < b.N; i++ {
if err := client.Enqueue(pool.Next()); err != nil {
enqueueErrors.Add(1)
}
}
b.StopTimer()
if failures := callback.FailureCount(); enqueueErrors.Load() > 0 || failures > 0 {
b.Fatalf("Benchmark invalid: %d enqueue errors, %d delivery failures",
enqueueErrors.Load(), failures)
}
}
// BenchmarkEnqueueThroughputWithCardinality measures enqueue speed at different cardinalities
// Uses production defaults for BatchSize, MaxEnqueuedRequests
func BenchmarkEnqueueThroughputWithCardinality(b *testing.B) {
cardinalities := []struct {
name string
cardinality PropertyCardinality
}{
{"low", CardinalityLow},
{"medium", CardinalityMedium},
{"high", CardinalityHigh},
}
for _, tc := range cardinalities {
b.Run(tc.name, func(b *testing.B) {
// Use cardinality-appropriate pool size with pre-cloned unique maps
pool := NewEventPoolWithDefaultSize(tc.cardinality)
callback := &BenchmarkCallback{}
var enqueueErrors atomic.Int64
client, _ := NewWithConfig("test-key", Config{
Transport: NoOpTransport(),
Callback: callback,
// Uses production defaults for BatchSize, MaxEnqueuedRequests
})
defer client.Close()
b.ResetTimer()
for i := 0; i < b.N; i++ {
if err := client.Enqueue(pool.Next()); err != nil {
enqueueErrors.Add(1)
}
}
b.StopTimer()
if failures := callback.FailureCount(); enqueueErrors.Load() > 0 || failures > 0 {
b.Fatalf("Benchmark invalid: %d enqueue errors, %d delivery failures",
enqueueErrors.Load(), failures)
}
})
}
}
// BenchmarkFeatureFlagLocalEvaluation measures flag evaluation performance
func BenchmarkFeatureFlagLocalEvaluation(b *testing.B) {
// Setup server that returns actual flag definitions
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case strings.HasPrefix(r.URL.Path, "/api/feature_flag/local_evaluation"):
w.Write([]byte(fixture("test-api-feature-flag.json")))
case strings.HasPrefix(r.URL.Path, "/batch"):
io.Copy(io.Discard, r.Body)
w.WriteHeader(200)
default:
w.WriteHeader(404)
}
}))
defer server.Close()
client, _ := NewWithConfig("test-key", Config{
PersonalApiKey: "test",
Endpoint: server.URL,
})
defer client.Close()
// Pre-warm cache - wait for flags to load
for i := 0; i < 10; i++ {
result, _ := client.GetFeatureFlag(FeatureFlagPayload{Key: "simpleFlag", DistinctId: "warmup"})
if result != nil {
break
}
}
distinctIds := make([]string, 1000)
for i := range distinctIds {
distinctIds[i] = fmt.Sprintf("user_%d", i)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
client.GetFeatureFlag(FeatureFlagPayload{
Key: "simpleFlag",
DistinctId: distinctIds[i%len(distinctIds)],
})
}
}
// BenchmarkBatchSizes measures throughput at various batch sizes
// Note: BatchSize=1 is excluded as it's pathological - creates one batch per event,
// overwhelming the worker pool with backpressure. Production min should be 10+.
func BenchmarkBatchSizes(b *testing.B) {
batchSizes := []int{10, 50, 100, 250, 500}
for _, batchSize := range batchSizes {
b.Run(fmt.Sprintf("batch_%d", batchSize), func(b *testing.B) {
pool := NewEventPool(b.N + 1000)
callback := &BenchmarkCallback{}
var enqueueErrors atomic.Int64
client, _ := NewWithConfig("test-key", Config{
Transport: NoOpTransport(),
Callback: callback,
BatchSize: batchSize,
})
defer client.Close()
b.ResetTimer()
for i := 0; i < b.N; i++ {
if err := client.Enqueue(pool.Next()); err != nil {
enqueueErrors.Add(1)
}
}
b.StopTimer()
if failures := callback.FailureCount(); enqueueErrors.Load() > 0 || failures > 0 {
b.Fatalf("Benchmark invalid: %d enqueue errors, %d delivery failures",
enqueueErrors.Load(), failures)
}
})
}
}
// BenchmarkEndToEndWithServer measures end-to-end throughput including HTTP
// Uses production defaults for BatchSize, MaxEnqueuedRequests
func BenchmarkEndToEndWithServer(b *testing.B) {
pool := NewEventPool(b.N + 1000)
server := httptest.NewServer(NoOpHandler())
defer server.Close()
callback := &BenchmarkCallback{}
var enqueueErrors atomic.Int64
client, _ := NewWithConfig("test-key", Config{
Endpoint: server.URL,
Callback: callback,
// Uses production defaults for BatchSize, MaxEnqueuedRequests
})
defer client.Close()
b.ResetTimer()
for i := 0; i < b.N; i++ {
if err := client.Enqueue(pool.Next()); err != nil {
enqueueErrors.Add(1)
}
}
b.StopTimer()
if failures := callback.FailureCount(); enqueueErrors.Load() > 0 || failures > 0 {
b.Fatalf("Benchmark invalid: %d enqueue errors, %d delivery failures",
enqueueErrors.Load(), failures)
}
}
// NoOpHandler returns an HTTP handler that discards the body and returns 200
func NoOpHandler() noOpHandler {
return noOpHandler{}
}
type noOpHandler struct{}
func (h noOpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
io.Copy(io.Discard, r.Body)
w.WriteHeader(200)
}
// BenchmarkEventPoolGeneration measures event pool creation time
func BenchmarkEventPoolGeneration(b *testing.B) {
sizes := []int{100, 1000, 10000}
for _, size := range sizes {
b.Run(fmt.Sprintf("size_%d", size), func(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
NewEventPool(size)
}
})
}
}
// BenchmarkEventPoolGenerationWithCardinality measures pool creation with cardinality
func BenchmarkEventPoolGenerationWithCardinality(b *testing.B) {
cardinalities := []struct {
name string
cardinality PropertyCardinality
}{
{"low", CardinalityLow},
{"medium", CardinalityMedium},
{"high", CardinalityHigh},
}
size := 100 // Keep small for high cardinality
for _, tc := range cardinalities {
b.Run(tc.name, func(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
NewEventPoolWithCardinality(size, tc.cardinality)
}
})
}
}
// BenchmarkValidation measures message validation overhead
func BenchmarkValidation(b *testing.B) {
capture := Capture{
DistinctId: "user_1",
Event: "test_event",
Properties: generateVariedProperties(42),
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
capture.Validate()
}
}
// BenchmarkAPIfy measures APIfy conversion overhead
func BenchmarkAPIfy(b *testing.B) {
cardinalities := []struct {
name string
cardinality PropertyCardinality
}{
{"low", CardinalityLow},
{"medium", CardinalityMedium},
{"high", CardinalityHigh},
}
for _, tc := range cardinalities {
b.Run(tc.name, func(b *testing.B) {
capture := Capture{
DistinctId: "user_1",
Event: "test_event",
Properties: generatePropertiesWithCardinality(42, tc.cardinality),
Groups: generateGroupsWithCardinality(42, tc.cardinality),
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
capture.APIfy()
}
})
}
}
// BenchmarkMessageEnqueueOverhead measures the overhead of the Enqueue path
func BenchmarkMessageEnqueueOverhead(b *testing.B) {
callback := &BenchmarkCallback{}
var enqueueErrors atomic.Int64
// Create a client with a very fast transport
client, _ := NewWithConfig("test-key", Config{
Transport: roundTripperFunc(func(r *http.Request) (*http.Response, error) {
io.Copy(io.Discard, r.Body)
return &http.Response{
StatusCode: 200,
Body: io.NopCloser(strings.NewReader("")),
}, nil
}),
Callback: callback,
BatchSize: 10000, // Large batch to avoid flushing during benchmark
})
defer client.Close()
// Pre-generate captures to avoid map race conditions
// Each iteration needs its own Properties map since APIfy iterates over it
captures := make([]Capture, b.N)
for i := range captures {
captures[i] = Capture{
DistinctId: "user_1",
Event: "test_event",
Properties: generateVariedProperties(i),
}
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
if err := client.Enqueue(captures[i]); err != nil {
enqueueErrors.Add(1)
}
}
b.StopTimer()
if failures := callback.FailureCount(); enqueueErrors.Load() > 0 || failures > 0 {
b.Fatalf("Benchmark invalid: %d enqueue errors, %d delivery failures",
enqueueErrors.Load(), failures)
}
}
+4 -1
View File
@@ -1,6 +1,8 @@
package posthog package posthog
import "time" import (
"time"
)
// SendFeatureFlagsValue defines the interface for feature flag configuration // SendFeatureFlagsValue defines the interface for feature flag configuration
type SendFeatureFlagsValue interface { type SendFeatureFlagsValue interface {
@@ -147,3 +149,4 @@ func (msg Capture) APIfy() APIMessage {
return apified return apified
} }
+1 -1
View File
@@ -1,11 +1,11 @@
package main package main
import ( import (
"encoding/json"
"fmt" "fmt"
"log" "log"
"os" "os"
json "github.com/goccy/go-json"
"github.com/posthog/posthog-go" "github.com/posthog/posthog-go"
"github.com/urfave/cli" "github.com/urfave/cli"
) )
+486
View File
@@ -0,0 +1,486 @@
package posthog
import (
"fmt"
json "github.com/goccy/go-json"
"net/http"
"net/http/httptest"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/require"
)
// TestConcurrentEnqueue tests thread-safety of the client under concurrent load
func TestConcurrentEnqueue(t *testing.T) {
t.Parallel()
goroutines := 20
eventsPerGoroutine := 50
totalEvents := goroutines * eventsPerGoroutine
pool := NewEventPool(totalEvents)
var received atomic.Int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var b batch
json.NewDecoder(r.Body).Decode(&b)
received.Add(int64(len(b.Messages)))
w.WriteHeader(200)
}))
defer server.Close()
client, err := NewWithConfig("test-key", Config{
Endpoint: server.URL,
// Uses production defaults for BatchSize, MaxEnqueuedRequests
})
require.NoError(t, err)
var wg sync.WaitGroup
for g := 0; g < goroutines; g++ {
wg.Add(1)
go func(goroutineID int) {
defer wg.Done()
for i := 0; i < eventsPerGoroutine; i++ {
idx := goroutineID*eventsPerGoroutine + i
err := client.Enqueue(pool.Get(idx))
if err != nil {
t.Errorf("Enqueue failed: %v", err)
}
}
}(g)
}
wg.Wait()
client.Close()
require.Equal(t, int64(totalEvents), received.Load(),
"All events should be delivered")
}
// TestConcurrentEnqueueDifferentMessageTypes tests concurrent enqueueing of different message types
func TestConcurrentEnqueueDifferentMessageTypes(t *testing.T) {
t.Parallel()
goroutines := 10
messagesPerGoroutine := 20
var received atomic.Int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var b batch
json.NewDecoder(r.Body).Decode(&b)
received.Add(int64(len(b.Messages)))
w.WriteHeader(200)
}))
defer server.Close()
client, err := NewWithConfig("test-key", Config{
Endpoint: server.URL,
// Uses production defaults for BatchSize, MaxEnqueuedRequests
})
require.NoError(t, err)
var wg sync.WaitGroup
for g := 0; g < goroutines; g++ {
wg.Add(1)
go func(goroutineID int) {
defer wg.Done()
for i := 0; i < messagesPerGoroutine; i++ {
msgType := i % 4
switch msgType {
case 0:
client.Enqueue(Capture{
DistinctId: fmt.Sprintf("user_%d", goroutineID),
Event: fmt.Sprintf("event_%d", i),
})
case 1:
client.Enqueue(Identify{
DistinctId: fmt.Sprintf("user_%d", goroutineID),
Properties: Properties{"name": fmt.Sprintf("User %d", goroutineID)},
})
case 2:
client.Enqueue(Alias{
DistinctId: fmt.Sprintf("user_%d", goroutineID),
Alias: fmt.Sprintf("alias_%d", goroutineID),
})
case 3:
client.Enqueue(GroupIdentify{
Type: "company",
Key: fmt.Sprintf("company_%d", goroutineID),
Properties: Properties{
"name": fmt.Sprintf("Company %d", goroutineID),
},
})
}
}
}(g)
}
wg.Wait()
client.Close()
expectedTotal := int64(goroutines * messagesPerGoroutine)
require.Equal(t, expectedTotal, received.Load(),
"All messages should be delivered")
}
// TestConcurrentFeatureFlagEvaluation tests concurrent flag evaluation with cache updates
func TestConcurrentFeatureFlagEvaluation(t *testing.T) {
t.Parallel()
// Setup mock server with feature flags
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(200)
// Return proper format for both local evaluation and flags endpoints
if r.URL.Path == "/api/feature_flag/local_evaluation" {
json.NewEncoder(w).Encode(map[string]interface{}{
"flags": []map[string]interface{}{
{
"id": 1,
"key": "test-flag",
"active": true,
"filters": map[string]interface{}{
"groups": []map[string]interface{}{
{
"rollout_percentage": 100,
},
},
},
},
},
"group_type_mapping": map[string]string{},
})
} else {
json.NewEncoder(w).Encode(map[string]interface{}{
"featureFlags": map[string]interface{}{
"test-flag": true,
},
})
}
}))
defer server.Close()
client, err := NewWithConfig("test-key", Config{
Endpoint: server.URL,
PersonalApiKey: "test-key",
})
require.NoError(t, err)
defer client.Close()
// Wait for initial flag fetch to complete
time.Sleep(100 * time.Millisecond)
goroutines := 100
evaluationsPerGoroutine := 50
var wg sync.WaitGroup
errorCount := atomic.Int64{}
for g := 0; g < goroutines; g++ {
wg.Add(1)
go func(goroutineID int) {
defer wg.Done()
for i := 0; i < evaluationsPerGoroutine; i++ {
distinctId := fmt.Sprintf("user_%d_%d", goroutineID, i)
_, err := client.GetFeatureFlag(FeatureFlagPayload{
Key: "test-flag",
DistinctId: distinctId,
})
if err != nil {
errorCount.Add(1)
}
}
}(g)
}
wg.Wait()
// With httptest.Server, there are no network issues - all evaluations should succeed
require.Equal(t, int64(0), errorCount.Load(),
"All feature flag evaluations should succeed - httptest.Server has no network issues")
}
// TestConcurrentClientOperations tests mixing Enqueue with Close operations
func TestConcurrentClientOperations(t *testing.T) {
t.Parallel()
// This tests that Close() properly waits for all pending operations
for iteration := 0; iteration < 5; iteration++ {
t.Run(fmt.Sprintf("iteration_%d", iteration), func(t *testing.T) {
var received atomic.Int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var b batch
json.NewDecoder(r.Body).Decode(&b)
received.Add(int64(len(b.Messages)))
w.WriteHeader(200)
}))
defer server.Close()
client, err := NewWithConfig("test-key", Config{
Endpoint: server.URL,
BatchSize: 50,
// Uses production defaults for MaxEnqueuedRequests
})
require.NoError(t, err)
// Start enqueueing
enqueueCount := 50
var wg sync.WaitGroup
for i := 0; i < enqueueCount; i++ {
wg.Add(1)
go func(idx int) {
defer wg.Done()
client.Enqueue(Capture{
DistinctId: fmt.Sprintf("user_%d", idx),
Event: "test_event",
})
}(i)
}
// Wait for all enqueues to complete
wg.Wait()
// Close should wait for all events to be sent
err = client.Close()
require.NoError(t, err)
// All events should have been delivered by the time Close returns
require.Equal(t, int64(enqueueCount), received.Load(),
"All events should be delivered before Close returns")
})
}
}
// TestConcurrentEnqueueWithSlowServer tests behavior with a slow server
func TestConcurrentEnqueueWithSlowServer(t *testing.T) {
goroutines := 20
eventsPerGoroutine := 10
totalEvents := goroutines * eventsPerGoroutine
pool := NewEventPool(totalEvents)
var received atomic.Int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Simulate slow server
time.Sleep(10 * time.Millisecond)
var b batch
json.NewDecoder(r.Body).Decode(&b)
received.Add(int64(len(b.Messages)))
w.WriteHeader(200)
}))
defer server.Close()
client, err := NewWithConfig("test-key", Config{
Endpoint: server.URL,
// Uses production defaults for BatchSize, MaxEnqueuedRequests
})
require.NoError(t, err)
var wg sync.WaitGroup
start := time.Now()
for g := 0; g < goroutines; g++ {
wg.Add(1)
go func(goroutineID int) {
defer wg.Done()
for i := 0; i < eventsPerGoroutine; i++ {
idx := goroutineID*eventsPerGoroutine + i
client.Enqueue(pool.Get(idx))
}
}(g)
}
wg.Wait()
client.Close()
elapsed := time.Since(start)
t.Logf("Slow server test: %d events in %v", totalEvents, elapsed)
require.Equal(t, int64(totalEvents), received.Load(),
"All events should be delivered even with slow server")
}
// TestConcurrentPrepareForSend tests that prepareForSend is thread-safe
func TestConcurrentPrepareForSend(t *testing.T) {
t.Parallel()
capture := Capture{
Type: "capture",
DistinctId: "test_user",
Event: "test_event",
Properties: generatePropertiesWithCardinality(42, CardinalityMedium),
Groups: generateGroupsWithCardinality(42, CardinalityMedium),
}
goroutines := 100
callsPerGoroutine := 100
var wg sync.WaitGroup
results := make([]int, goroutines)
for g := 0; g < goroutines; g++ {
wg.Add(1)
go func(goroutineID int) {
defer wg.Done()
var total int
for i := 0; i < callsPerGoroutine; i++ {
data, _, err := prepareForSend(capture)
if err != nil {
t.Errorf("prepareForSend error: %v", err)
return
}
total += len(data)
}
results[goroutineID] = total / callsPerGoroutine
}(g)
}
wg.Wait()
// All goroutines should get the same average result
first := results[0]
for i, result := range results {
if result != first {
t.Errorf("Inconsistent prepareForSend size: goroutine 0 got %d, goroutine %d got %d",
first, i, result)
}
}
}
// TestConcurrentEventPoolAccess tests thread-safe access to EventPool
func TestConcurrentEventPoolAccess(t *testing.T) {
t.Parallel()
poolSize := 1000
pool := NewEventPool(poolSize)
goroutines := 50
accessesPerGoroutine := 200
var wg sync.WaitGroup
errorCount := atomic.Int64{}
for g := 0; g < goroutines; g++ {
wg.Add(1)
go func() {
defer wg.Done()
for i := 0; i < accessesPerGoroutine; i++ {
event := pool.Next()
if event.DistinctId == "" || event.Event == "" {
errorCount.Add(1)
}
}
}()
}
wg.Wait()
require.Equal(t, int64(0), errorCount.Load(),
"No errors should occur during concurrent pool access")
}
// TestConcurrentCallbackExecution tests that callbacks are executed thread-safely
func TestConcurrentCallbackExecution(t *testing.T) {
t.Parallel()
callback := NewUnifiedCallback(t)
var received atomic.Int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var b batch
json.NewDecoder(r.Body).Decode(&b)
received.Add(int64(len(b.Messages)))
w.WriteHeader(200)
}))
defer server.Close()
client, err := NewWithConfig("test-key", Config{
Endpoint: server.URL,
BatchSize: 50,
Callback: callback,
// Uses production defaults for MaxEnqueuedRequests
})
require.NoError(t, err)
goroutines := 10
eventsPerGoroutine := 20
var wg sync.WaitGroup
for g := 0; g < goroutines; g++ {
wg.Add(1)
go func(goroutineID int) {
defer wg.Done()
for i := 0; i < eventsPerGoroutine; i++ {
client.Enqueue(Capture{
DistinctId: fmt.Sprintf("user_%d", goroutineID),
Event: fmt.Sprintf("event_%d", i),
})
}
}(g)
}
wg.Wait()
client.Close()
successCount, failureCount := callback.GetCounts()
t.Logf("Callback counts: success=%d, failure=%d", successCount, failureCount)
// All events should have been delivered with zero failures
totalEvents := int64(goroutines * eventsPerGoroutine)
require.Equal(t, totalEvents, received.Load(),
"All events should be delivered")
require.Equal(t, 0, failureCount,
"No callback failures should occur")
}
// TestRaceConditionDetection is specifically designed to trigger race detection
// Run with: go test -race -run TestRaceConditionDetection
func TestRaceConditionDetection(t *testing.T) {
t.Parallel()
t.Run("concurrent_enqueue_and_close", func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(200)
}))
defer server.Close()
for i := 0; i < 10; i++ {
client, _ := NewWithConfig("test-key", Config{
Endpoint: server.URL,
// Uses production defaults for BatchSize, MaxEnqueuedRequests
})
var wg sync.WaitGroup
// Concurrent enqueues
for j := 0; j < 20; j++ {
wg.Add(1)
go func(idx int) {
defer wg.Done()
client.Enqueue(Capture{
DistinctId: fmt.Sprintf("user_%d", idx),
Event: "test",
})
}(j)
}
wg.Wait()
client.Close()
}
})
t.Run("concurrent_pool_access", func(t *testing.T) {
pool := NewEventPool(100)
var wg sync.WaitGroup
for i := 0; i < 100; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for j := 0; j < 50; j++ {
_ = pool.Next()
}
}()
}
wg.Wait()
})
}
+55 -14
View File
@@ -29,7 +29,7 @@ type Config struct {
// The flushing interval of the client. Messages will be sent when they've // The flushing interval of the client. Messages will be sent when they've
// been queued up to the maximum batch size or when the flushing interval // been queued up to the maximum batch size or when the flushing interval
// timer triggers. // timer triggers. If zero, defaults to 5 seconds.
Interval time.Duration Interval time.Duration
// Interval at which to fetch new feature flag definitions, 5min by default // Interval at which to fetch new feature flag definitions, 5min by default
@@ -69,7 +69,7 @@ type Config struct {
// The maximum number of messages that will be sent in one API call. // The maximum number of messages that will be sent in one API call.
// Messages will be sent when they've been queued up to the maximum batch // Messages will be sent when they've been queued up to the maximum batch
// size or when the flushing interval timer triggers. // size or when the flushing interval timer triggers. If zero, defaults to 250.
// Note that the API will still enforce a 500KB limit on each HTTP request // Note that the API will still enforce a 500KB limit on each HTTP request
// which is independent from the number of embedded messages. // which is independent from the number of embedded messages.
BatchSize int BatchSize int
@@ -85,19 +85,37 @@ type Config struct {
// If not set the client will fallback to use a default retry policy. // If not set the client will fallback to use a default retry policy.
RetryAfter func(int) time.Duration RetryAfter func(int) time.Duration
// MaxRetries is a maximum number of a request is retried on failure. // MaxRetries is the maximum number of times a request is retried on failure.
// Must be in range [0,9]. If nil, defaults to 9 (10 total attempts).
MaxRetries *int MaxRetries *int
// ShutdownTimeout is the maximum time to wait for in-flight messages
// to be sent during Close(). If zero or negative, waits indefinitely
// (preserving backward-compatible behavior). Set to a positive duration
// to enable timeout-based shutdown.
ShutdownTimeout time.Duration
// BatchUploadTimeout is the timeout for uploading batched events to the
// /batch/ endpoint. If zero, defaults to 10 seconds.
BatchUploadTimeout time.Duration
// BatchSubmitTimeout is the maximum time to wait when submitting a batch
// to the worker pool if the queue is full. This provides backpressure
// smoothing during transient backend latency spikes, reducing data loss.
// If zero, defaults to 100ms. Set to a negative value for non-blocking
// behavior (immediate drop when queue is full).
BatchSubmitTimeout time.Duration
// MaxEnqueuedRequests is the maximum number of batches that can be queued
// for sending. When the queue is full, new batches are dropped and the
// failure callback is invoked. If zero, defaults to 1000.
MaxEnqueuedRequests int
// A function called by the client to get the current time, `time.Now` is // A function called by the client to get the current time, `time.Now` is
// used by default. // used by default.
// This field is not exported and only exposed internally to control concurrency. // This field is not exported and only exposed internally to control concurrency.
now func() time.Time now func() time.Time
// The maximum number of goroutines that will be spawned by a client to send
// requests to the backend API.
// This field is not exported and only exposed internally to control concurrency.
maxConcurrentRequests int
// maxAttempts is a maximum numbers we try to send data to capture endpoint, must be in range [1,10]. // maxAttempts is a maximum numbers we try to send data to capture endpoint, must be in range [1,10].
maxAttempts int maxAttempts int
} }
@@ -128,6 +146,19 @@ const (
// DefaultBatchSize sets the default batch size used by client instances if none // DefaultBatchSize sets the default batch size used by client instances if none
// was explicitly set. // was explicitly set.
DefaultBatchSize = 250 DefaultBatchSize = 250
// DefaultBatchUploadTimeout is the default timeout for uploading batched
// events to the /batch/ endpoint.
DefaultBatchUploadTimeout = 10 * time.Second
// DefaultBatchSubmitTimeout is the default timeout for submitting batches
// to the worker pool when the queue is full. This allows workers time to
// complete during transient latency spikes, reducing unnecessary data loss.
DefaultBatchSubmitTimeout = 100 * time.Millisecond
// DefaultMaxEnqueuedRequests is the default maximum number of batches that
// can be queued for sending.
DefaultMaxEnqueuedRequests = 1000
) )
// Validate verifies that fields that don't have zero-values are set to valid values, // Validate verifies that fields that don't have zero-values are set to valid values,
@@ -157,7 +188,7 @@ func (c *Config) Validate() error {
} }
} }
if c.MaxRetries != nil && (*c.MaxRetries < 0 || 9 < *c.MaxRetries) { if c.MaxRetries != nil && (*c.MaxRetries < 0 || *c.MaxRetries > 9) {
return ConfigError{ return ConfigError{
Reason: "max retries out of range [0,9]", Reason: "max retries out of range [0,9]",
Field: "MaxRetries", Field: "MaxRetries",
@@ -187,9 +218,8 @@ func makeConfig(c Config) Config {
c.FeatureFlagRequestTimeout = DefaultFeatureFlagRequestTimeout c.FeatureFlagRequestTimeout = DefaultFeatureFlagRequestTimeout
} }
if c.Transport == nil { // Note: c.Transport == nil is handled by makeHttpClient() which clones
c.Transport = http.DefaultTransport // DefaultTransport with tuned connection pool settings
}
if c.Logger == nil { if c.Logger == nil {
c.Logger = newDefaultLogger(c.Verbose) c.Logger = newDefaultLogger(c.Verbose)
@@ -213,8 +243,19 @@ func makeConfig(c Config) Config {
c.maxAttempts = 10 c.maxAttempts = 10
} }
if c.maxConcurrentRequests == 0 { // Note: ShutdownTimeout == 0 means wait indefinitely (backward compatible).
c.maxConcurrentRequests = 1000 // Users opt-in to timeout by setting a positive duration.
if c.BatchUploadTimeout == 0 {
c.BatchUploadTimeout = DefaultBatchUploadTimeout
}
if c.BatchSubmitTimeout == 0 {
c.BatchSubmitTimeout = DefaultBatchSubmitTimeout
}
if c.MaxEnqueuedRequests == 0 {
c.MaxEnqueuedRequests = DefaultMaxEnqueuedRequests
} }
if c.GetDisableGeoIP() { if c.GetDisableGeoIP() {
+4
View File
@@ -57,4 +57,8 @@ var (
// failed because the JSON representation of a message exceeded the upper // failed because the JSON representation of a message exceeded the upper
// limit. // limit.
ErrMessageTooBig = errors.New("the message exceeds the maximum allowed size") ErrMessageTooBig = errors.New("the message exceeds the maximum allowed size")
// ErrNoPersonalAPIKey is returned when one tries to use feature flags
// without specifying a PersonalAPIKey.
ErrNoPersonalAPIKey = errors.New("no PersonalAPIKey provided")
) )
+3 -1
View File
@@ -1,6 +1,8 @@
package posthog package posthog
import "time" import (
"time"
)
var _ Message = (*Exception)(nil) var _ Message = (*Exception)(nil)
+1 -1
View File
@@ -1,10 +1,10 @@
package main package main
import ( import (
"encoding/json"
"fmt" "fmt"
"time" "time"
json "github.com/goccy/go-json"
"github.com/posthog/posthog-go" "github.com/posthog/posthog-go"
) )
-43
View File
@@ -1,43 +0,0 @@
package posthog
import "sync"
type executor struct {
m sync.Mutex
available int
closed bool
}
func newExecutor(cap int) *executor {
e := &executor{
available: cap,
}
return e
}
func (e *executor) do(task func()) bool {
e.m.Lock()
defer e.m.Unlock()
if e.closed || e.available <= 0 {
return false
}
e.available--
go func() {
defer func() {
e.m.Lock()
defer e.m.Unlock()
e.available++
}()
task()
}()
return true
}
func (e *executor) close() {
e.m.Lock()
defer e.m.Unlock()
e.closed = true
}
-56
View File
@@ -1,56 +0,0 @@
package posthog
import (
"sync"
"testing"
"time"
)
func TestExecutorClose(t *testing.T) {
// Simply make sure that nothing raises a panic nor blocks.
ex := newExecutor(1)
ex.close()
}
func TestExecutorSimple(t *testing.T) {
wg := &sync.WaitGroup{}
ex := newExecutor(1)
defer ex.close()
wg.Add(1)
if !ex.do(wg.Done) {
t.Error("failed pushing a task to an executor with a capacity of 1")
return
}
// Make sure wg.Done gets called, this shouldn't block indefinitely.
wg.Wait()
}
func TestExecutorMulti(t *testing.T) {
wg := &sync.WaitGroup{}
ex := newExecutor(3)
defer ex.close()
// Schedule a couple of tasks to fill the executor.
for i := 0; i != 3; i++ {
wg.Add(1)
if !ex.do(func() {
time.Sleep(10 * time.Millisecond)
wg.Done()
}) {
t.Error("failed pushing a task to an executor with a capacity of 1")
return
}
}
// Make sure the executor refuses more tasks.
if ex.do(func() {}) {
t.Error("the executor should have been full and refused to run more tasks")
}
// Make sure wg.Done gets called, this shouldn't block indefinitely.
wg.Wait()
}
+95
View File
@@ -0,0 +1,95 @@
package posthog
import (
"net/http"
"net/http/httptest"
"testing"
)
func TestComplexCohortsLocally(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(fixture("feature_flag/test-complex-cohorts-locally.json"))) // Don't return anything for local eval
}))
defer server.Close()
client, _ := NewWithConfig("Csyjlnlun3OzyNJAafdlv", Config{
PersonalApiKey: "some very secret key",
Endpoint: server.URL,
})
defer client.Close()
payload := FeatureFlagPayload{
Key: "beta-feature",
DistinctId: "some-distinct-id",
PersonProperties: NewProperties().Set("region", "UK"),
}
isMatch, err := client.IsFeatureEnabled(payload)
if err != nil {
t.Fatal(err)
}
if isMatch != false {
t.Error("Should not match")
}
payload.PersonProperties = NewProperties().Set("region", "USA").Set("other", "thing")
isMatch, _ = client.IsFeatureEnabled(payload)
if isMatch != true {
t.Error("Should match")
}
// even though 'other' property is not present, the cohort should still match since it's an OR condition
payload.PersonProperties = NewProperties().Set("region", "USA").Set("nation", "UK")
isMatch, _ = client.IsFeatureEnabled(payload)
if isMatch != true {
t.Error("Should match")
}
}
func TestComplexCohortsWithNegationLocally(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(fixture("feature_flag/test-complex-cohorts-negation-locally.json"))) // Don't return anything for local eval
}))
defer server.Close()
client, _ := NewWithConfig("Csyjlnlun3OzyNJAafdlv", Config{
PersonalApiKey: "some very secret key",
Endpoint: server.URL,
})
defer client.Close()
payload := FeatureFlagPayload{
Key: "beta-feature",
DistinctId: "some-distinct-id",
PersonProperties: NewProperties().Set("region", "UK"),
}
isMatch, err := client.IsFeatureEnabled(payload)
if err != nil {
t.Fatal(err)
}
if isMatch != false {
t.Error("Should not match")
}
// even though 'other' property is not present, the cohort should still match since it's an OR condition
payload.PersonProperties = NewProperties().Set("region", "USA").Set("nation", "UK")
isMatch, _ = client.IsFeatureEnabled(payload)
if isMatch != true {
t.Error("Should match")
}
// # since 'other' is negated, we return False. Since 'nation' is not present, we can't tell whether the flag should be true or false, so go to decide
payload.PersonProperties = NewProperties().Set("region", "USA").Set("other", "thing")
_, err = client.IsFeatureEnabled(payload)
if err != nil {
t.Error("Expected to fail")
}
payload.PersonProperties = NewProperties().Set("region", "USA").Set("other", "thing2")
isMatch, _ = client.IsFeatureEnabled(payload)
if isMatch != true {
t.Error("Should match")
}
}
+692
View File
@@ -0,0 +1,692 @@
package posthog
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/stretchr/testify/require"
)
func TestFlagDependenciesSimpleChain(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasPrefix(r.URL.Path, "/api/feature_flag/local_evaluation") {
w.Write([]byte(`{
"flags": [
{
"id": 1,
"name": "Flag A",
"key": "flag-a",
"active": true,
"filters": {
"groups": [
{
"properties": [
{
"key": "email",
"operator": "icontains",
"value": "@example.com",
"type": "person"
}
],
"rollout_percentage": 100
}
]
}
},
{
"id": 2,
"name": "Flag B",
"key": "flag-b",
"active": true,
"filters": {
"groups": [
{
"properties": [
{
"key": "flag-a",
"operator": "flag_evaluates_to",
"value": true,
"type": "flag",
"dependency_chain": ["flag-a"]
}
],
"rollout_percentage": 100
}
]
}
}
],
"group_type_mapping": {}
}`))
}
}))
defer server.Close()
client, _ := NewWithConfig("test-api-key", Config{
PersonalApiKey: "test-personal-api-key",
Endpoint: server.URL,
})
defer client.Close()
// Test when dependency is satisfied
result, err := client.IsFeatureEnabled(
FeatureFlagPayload{
Key: "flag-b",
DistinctId: "test-user",
PersonProperties: NewProperties().Set("email", "[email protected]"),
OnlyEvaluateLocally: true,
},
)
require.NoError(t, err)
require.Equal(t, true, result)
// Test flag-a directly with email that doesn't match - should be false
flagAResult, err := client.IsFeatureEnabled(
FeatureFlagPayload{
Key: "flag-a",
DistinctId: "test-user-2",
PersonProperties: NewProperties().Set("email", "[email protected]"),
OnlyEvaluateLocally: true,
},
)
require.NoError(t, err)
require.Equal(t, false, flagAResult, "flag-a should be false for email that doesn't contain @example.com")
// Test when dependency is not satisfied
result, err = client.IsFeatureEnabled(
FeatureFlagPayload{
Key: "flag-b",
DistinctId: "test-user-2",
PersonProperties: NewProperties().Set("email", "[email protected]"),
OnlyEvaluateLocally: true,
},
)
require.NoError(t, err)
require.Equal(t, false, result)
}
func TestFlagDependenciesCircularDependency(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasPrefix(r.URL.Path, "/api/feature_flag/local_evaluation") {
w.Write([]byte(`{
"flags": [
{
"id": 1,
"name": "Flag A",
"key": "flag-a",
"active": true,
"filters": {
"groups": [
{
"properties": [
{
"key": "flag-b",
"operator": "flag_evaluates_to",
"value": true,
"type": "flag",
"dependency_chain": []
}
],
"rollout_percentage": 100
}
]
}
},
{
"id": 2,
"name": "Flag B",
"key": "flag-b",
"active": true,
"filters": {
"groups": [
{
"properties": [
{
"key": "flag-a",
"operator": "flag_evaluates_to",
"value": true,
"type": "flag",
"dependency_chain": []
}
],
"rollout_percentage": 100
}
]
}
}
],
"group_type_mapping": {}
}`))
} else if strings.HasPrefix(r.URL.Path, "/flags/") {
w.Write([]byte(`{"featureFlags": {}}`))
}
}))
defer server.Close()
client, _ := NewWithConfig("test-api-key", Config{
PersonalApiKey: "test-personal-api-key",
Endpoint: server.URL,
})
defer client.Close()
// Both flags should fall back to remote evaluation due to circular dependency
result, err := client.IsFeatureEnabled(
FeatureFlagPayload{
Key: "flag-a",
DistinctId: "test-user",
},
)
require.NoError(t, err)
require.Equal(t, false, result)
result, err = client.IsFeatureEnabled(
FeatureFlagPayload{
Key: "flag-b",
DistinctId: "test-user",
},
)
require.NoError(t, err)
require.Equal(t, false, result)
}
func TestFlagDependenciesMissingFlag(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasPrefix(r.URL.Path, "/api/feature_flag/local_evaluation") {
w.Write([]byte(`{
"flags": [
{
"id": 1,
"name": "Flag A",
"key": "flag-a",
"active": true,
"filters": {
"groups": [
{
"properties": [
{
"key": "non-existent-flag",
"operator": "flag_evaluates_to",
"value": true,
"type": "flag",
"dependency_chain": ["non-existent-flag"]
}
],
"rollout_percentage": 100
}
]
}
}
],
"group_type_mapping": {}
}`))
} else if strings.HasPrefix(r.URL.Path, "/flags/") {
w.Write([]byte(`{"featureFlags": {}}`))
}
}))
defer server.Close()
client, _ := NewWithConfig("test-api-key", Config{
PersonalApiKey: "test-personal-api-key",
Endpoint: server.URL,
})
defer client.Close()
// Should fall back to remote evaluation because dependency doesn't exist
result, err := client.IsFeatureEnabled(
FeatureFlagPayload{
Key: "flag-a",
DistinctId: "test-user",
},
)
require.NoError(t, err)
require.Equal(t, false, result)
}
func TestFlagDependenciesComplexChain(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasPrefix(r.URL.Path, "/api/feature_flag/local_evaluation") {
w.Write([]byte(`{
"flags": [
{
"id": 1,
"name": "Flag A",
"key": "flag-a",
"active": true,
"filters": {
"groups": [
{
"properties": [],
"rollout_percentage": 100
}
]
}
},
{
"id": 2,
"name": "Flag B",
"key": "flag-b",
"active": true,
"filters": {
"groups": [
{
"properties": [],
"rollout_percentage": 100
}
]
}
},
{
"id": 3,
"name": "Flag C",
"key": "flag-c",
"active": true,
"filters": {
"groups": [
{
"properties": [
{
"key": "flag-a",
"operator": "flag_evaluates_to",
"value": true,
"type": "flag",
"dependency_chain": ["flag-a"]
},
{
"key": "flag-b",
"operator": "flag_evaluates_to",
"value": true,
"type": "flag",
"dependency_chain": ["flag-b"]
}
],
"rollout_percentage": 100
}
]
}
},
{
"id": 4,
"name": "Flag D",
"key": "flag-d",
"active": true,
"filters": {
"groups": [
{
"properties": [
{
"key": "flag-c",
"operator": "flag_evaluates_to",
"value": true,
"type": "flag",
"dependency_chain": ["flag-a", "flag-b", "flag-c"]
}
],
"rollout_percentage": 100
}
]
}
}
],
"group_type_mapping": {}
}`))
}
}))
defer server.Close()
client, _ := NewWithConfig("test-api-key", Config{
PersonalApiKey: "test-personal-api-key",
Endpoint: server.URL,
})
defer client.Close()
// All dependencies satisfied - should return true
result, err := client.IsFeatureEnabled(
FeatureFlagPayload{
Key: "flag-d",
DistinctId: "test-user",
OnlyEvaluateLocally: true,
},
)
require.NoError(t, err)
require.Equal(t, true, result)
}
func TestFlagDependenciesMixedConditions(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasPrefix(r.URL.Path, "/api/feature_flag/local_evaluation") {
w.Write([]byte(`{
"flags": [
{
"id": 1,
"name": "Base Flag",
"key": "base-flag",
"active": true,
"filters": {
"groups": [
{
"properties": [],
"rollout_percentage": 100
}
]
}
},
{
"id": 2,
"name": "Mixed Flag",
"key": "mixed-flag",
"active": true,
"filters": {
"groups": [
{
"properties": [
{
"key": "base-flag",
"operator": "flag_evaluates_to",
"value": true,
"type": "flag",
"dependency_chain": ["base-flag"]
},
{
"key": "email",
"operator": "icontains",
"value": "@example.com",
"type": "person"
}
],
"rollout_percentage": 100
}
]
}
}
],
"group_type_mapping": {}
}`))
}
}))
defer server.Close()
client, _ := NewWithConfig("test-api-key", Config{
PersonalApiKey: "test-personal-api-key",
Endpoint: server.URL,
})
defer client.Close()
// Both flag dependency and email condition satisfied
result, err := client.IsFeatureEnabled(
FeatureFlagPayload{
Key: "mixed-flag",
DistinctId: "test-user",
PersonProperties: NewProperties().Set("email", "[email protected]"),
OnlyEvaluateLocally: true,
},
)
require.NoError(t, err)
require.Equal(t, true, result)
// Flag dependency satisfied but email condition not satisfied
result, err = client.IsFeatureEnabled(
FeatureFlagPayload{
Key: "mixed-flag",
DistinctId: "test-user-2",
PersonProperties: NewProperties().Set("email", "[email protected]"),
OnlyEvaluateLocally: true,
},
)
require.NoError(t, err)
require.Equal(t, false, result)
}
func TestFlagDependenciesMalformedChain(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasPrefix(r.URL.Path, "/api/feature_flag/local_evaluation") {
w.Write([]byte(`{
"flags": [
{
"id": 1,
"name": "Base Flag",
"key": "base-flag",
"active": true,
"filters": {
"groups": [
{
"properties": [],
"rollout_percentage": 100
}
]
}
},
{
"id": 2,
"name": "Missing Chain Flag",
"key": "missing-chain-flag",
"active": true,
"filters": {
"groups": [
{
"properties": [
{
"key": "base-flag",
"operator": "flag_evaluates_to",
"value": true,
"type": "flag"
}
],
"rollout_percentage": 100
}
]
}
}
],
"group_type_mapping": {}
}`))
} else if strings.HasPrefix(r.URL.Path, "/flags/") {
w.Write([]byte(`{"featureFlags": {}}`))
}
}))
defer server.Close()
client, _ := NewWithConfig("test-api-key", Config{
PersonalApiKey: "test-personal-api-key",
Endpoint: server.URL,
})
defer client.Close()
// Should fall back to remote evaluation when dependency_chain is missing
result, err := client.IsFeatureEnabled(
FeatureFlagPayload{
Key: "missing-chain-flag",
DistinctId: "test-user",
},
)
require.NoError(t, err)
require.Equal(t, false, result)
}
// TestMultiLevelMultivariateDependencyChain tests multi-level dependency chains with multivariate flags
// This test is equivalent to the EvaluatesMultiLevelMultivariateDependencyChain test in the .NET SDK
func TestMultiLevelMultivariateDependencyChain(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasPrefix(r.URL.Path, "/api/feature_flag/local_evaluation") {
w.Write([]byte(`{
"flags": [
{
"id": 1,
"name": "Leaf Flag",
"key": "leaf-flag",
"active": true,
"filters": {
"groups": [
{
"properties": [
{
"key": "email",
"operator": "icontains",
"value": "control",
"type": "person"
}
],
"rollout_percentage": 100,
"variant": "control"
},
{
"properties": [
{
"key": "email",
"operator": "icontains",
"value": "test",
"type": "person"
}
],
"rollout_percentage": 100,
"variant": "test"
}
],
"multivariate": {
"variants": [
{"key": "control", "name": "Control", "rollout_percentage": 50},
{"key": "test", "name": "Test", "rollout_percentage": 50}
]
}
}
},
{
"id": 2,
"name": "Intermediate Flag",
"key": "intermediate-flag",
"active": true,
"filters": {
"groups": [
{
"properties": [
{
"key": "leaf-flag",
"operator": "flag_evaluates_to",
"value": "control",
"type": "flag",
"dependency_chain": ["leaf-flag"]
}
],
"rollout_percentage": 100,
"variant": "blue"
}
],
"multivariate": {
"variants": [
{"key": "blue", "name": "Blue", "rollout_percentage": 60},
{"key": "green", "name": "Green", "rollout_percentage": 40}
]
}
}
},
{
"id": 3,
"name": "Dependent Flag",
"key": "dependent-flag",
"active": true,
"filters": {
"groups": [
{
"properties": [
{
"key": "intermediate-flag",
"operator": "flag_evaluates_to",
"value": "blue",
"type": "flag",
"dependency_chain": ["leaf-flag", "intermediate-flag"]
}
],
"rollout_percentage": 100
}
]
}
}
],
"group_type_mapping": {}
}`))
} else if strings.HasPrefix(r.URL.Path, "/flags/") {
w.Write([]byte(`{"featureFlags": {}}`))
}
}))
defer server.Close()
client, _ := NewWithConfig("test-api-key", Config{
PersonalApiKey: "test-personal-api-key",
Endpoint: server.URL,
})
defer client.Close()
// Test leaf flag evaluates to "control" variant for [email protected]
result, err := client.GetFeatureFlag(
FeatureFlagPayload{
Key: "leaf-flag",
DistinctId: "user-control",
PersonProperties: NewProperties().
Set("email", "[email protected]"),
},
)
require.NoError(t, err)
require.Equal(t, "control", result)
// Test leaf flag evaluates to "test" variant for [email protected]
result, err = client.GetFeatureFlag(
FeatureFlagPayload{
Key: "leaf-flag",
DistinctId: "user-test",
PersonProperties: NewProperties().
Set("email", "[email protected]"),
},
)
require.NoError(t, err)
require.Equal(t, "test", result)
// Test intermediate flag evaluates to "blue" when leaf-flag is "control"
result, err = client.GetFeatureFlag(
FeatureFlagPayload{
Key: "intermediate-flag",
DistinctId: "user-control",
PersonProperties: NewProperties().
Set("email", "[email protected]"),
},
)
require.NoError(t, err)
require.Equal(t, "blue", result)
// Test intermediate flag evaluates to false when leaf-flag is "test" (dependency not met)
resultBool, err := client.IsFeatureEnabled(
FeatureFlagPayload{
Key: "intermediate-flag",
DistinctId: "user-test",
PersonProperties: NewProperties().
Set("email", "[email protected]"),
},
)
require.NoError(t, err)
require.Equal(t, false, resultBool)
// Test dependent flag is true when leaf-flag="control" and intermediate-flag="blue"
resultBool, err = client.IsFeatureEnabled(
FeatureFlagPayload{
Key: "dependent-flag",
DistinctId: "user-control",
PersonProperties: NewProperties().
Set("email", "[email protected]"),
},
)
require.NoError(t, err)
require.Equal(t, true, resultBool)
// Test dependent flag is false when leaf-flag="test" (breaks dependency chain)
resultBool, err = client.IsFeatureEnabled(
FeatureFlagPayload{
Key: "dependent-flag",
DistinctId: "user-test",
PersonProperties: NewProperties().
Set("email", "[email protected]"),
},
)
require.NoError(t, err)
require.Equal(t, false, resultBool)
}
+101 -35
View File
@@ -4,32 +4,69 @@ import (
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"strings" "strings"
"sync"
"time"
"testing" "testing"
) )
// eventCapture is a test helper that captures events via the Callback interface
type eventCapture struct {
mu sync.Mutex
events []CaptureInApi
}
func (e *eventCapture) Success(m APIMessage) {
if capture, ok := m.(CaptureInApi); ok {
e.mu.Lock()
e.events = append(e.events, capture)
e.mu.Unlock()
}
}
func (e *eventCapture) Failure(m APIMessage, err error) {}
func (e *eventCapture) getLastEvent() *CaptureInApi {
e.mu.Lock()
defer e.mu.Unlock()
if len(e.events) == 0 {
return nil
}
return &e.events[len(e.events)-1]
}
func (e *eventCapture) waitForEvent(timeout time.Duration) *CaptureInApi {
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
if event := e.getLastEvent(); event != nil {
return event
}
time.Sleep(10 * time.Millisecond)
}
return nil
}
/* /*
Tests of the feature flag api against the flags endpoint, no local evaluation. Tests of the feature flag api against the flags endpoint, no local evaluation.
This is primarily here to ensure we handle the different versions of the flags This is primarily here to ensure we handle the different versions of the flags
endpoint correctly. endpoint correctly.
*/ */
func TestFlags(t *testing.T) { func TestFlags(t *testing.T) {
validateCapturedEvent := func(t *testing.T, client Client) { t.Parallel()
lastEvent := client.GetLastCapturedEvent() validateCapturedEvent := func(t *testing.T, event *CaptureInApi) {
if event == nil {
if lastEvent == nil {
return return
} }
if lastEvent.Event != "$feature_flag_called" { if event.Event != "$feature_flag_called" {
t.Errorf("Expected a $feature_flag_called event, got: %v", lastEvent) t.Errorf("Expected a $feature_flag_called event, got: %v", event.Event)
} }
if lastEvent.Properties["$feature_flag_request_id"] != "42853c54-1431-4861-996e-3a548989fa2c" { if event.Properties["$feature_flag_request_id"] != "42853c54-1431-4861-996e-3a548989fa2c" {
t.Errorf("Expected $feature_flag_request_id property to be 42853c54-1431-4861-996e-3a548989fa2c, got: %v", lastEvent.Properties["$feature_flag_request_id"]) t.Errorf("Expected $feature_flag_request_id property to be 42853c54-1431-4861-996e-3a548989fa2c, got: %v", event.Properties["$feature_flag_request_id"])
} }
if lastEvent.Properties["$feature_flag_evaluated_at"] != int64(1737312368000) { if event.Properties["$feature_flag_evaluated_at"] != int64(1737312368000) {
t.Errorf("Expected $feature_flag_evaluated_at property to be 1737312368000, got: %v", lastEvent.Properties["$feature_flag_evaluated_at"]) t.Errorf("Expected $feature_flag_evaluated_at property to be 1737312368000, got: %v", event.Properties["$feature_flag_evaluated_at"])
} }
} }
@@ -42,6 +79,7 @@ func TestFlags(t *testing.T) {
} }
for _, test := range tests { for _, test := range tests {
test := test // Capture loop variable for Go 1.21 compatibility
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasPrefix(r.URL.Path, "/flags") { if strings.HasPrefix(r.URL.Path, "/flags") {
w.Write([]byte(fixture(test.fixture))) w.Write([]byte(fixture(test.fixture)))
@@ -62,8 +100,11 @@ func TestFlags(t *testing.T) {
for _, subTest := range subTests { for _, subTest := range subTests {
t.Run(subTest.name+" "+subTest.flagKey, func(t *testing.T) { t.Run(subTest.name+" "+subTest.flagKey, func(t *testing.T) {
capture := &eventCapture{}
client, _ := NewWithConfig("Csyjlnlun3OzyNJAafdlv", Config{ client, _ := NewWithConfig("Csyjlnlun3OzyNJAafdlv", Config{
Endpoint: server.URL, Endpoint: server.URL,
BatchSize: 1, // Send immediately
Callback: capture,
}) })
defer client.Close() defer client.Close()
@@ -79,7 +120,8 @@ func TestFlags(t *testing.T) {
t.Errorf("Expected IsFeatureEnabled to return %v but was %v", subTest.expected, isMatch) t.Errorf("Expected IsFeatureEnabled to return %v but was %v", subTest.expected, isMatch)
} }
validateCapturedEvent(t, client) event := capture.waitForEvent(time.Second)
validateCapturedEvent(t, event)
}) })
} }
}) })
@@ -98,8 +140,11 @@ func TestFlags(t *testing.T) {
for _, subTest := range subTests { for _, subTest := range subTests {
t.Run(subTest.name+" "+subTest.flagKey, func(t *testing.T) { t.Run(subTest.name+" "+subTest.flagKey, func(t *testing.T) {
capture := &eventCapture{}
client, _ := NewWithConfig("Csyjlnlun3OzyNJAafdlv", Config{ client, _ := NewWithConfig("Csyjlnlun3OzyNJAafdlv", Config{
Endpoint: server.URL, Endpoint: server.URL,
BatchSize: 1, // Send immediately
Callback: capture,
}) })
defer client.Close() defer client.Close()
@@ -115,7 +160,8 @@ func TestFlags(t *testing.T) {
t.Errorf("Expected GetFeatureFlag to return %v but was %v", subTest.expected, flag) t.Errorf("Expected GetFeatureFlag to return %v but was %v", subTest.expected, flag)
} }
validateCapturedEvent(t, client) event := capture.waitForEvent(time.Second)
validateCapturedEvent(t, event)
}) })
} }
}) })
@@ -134,8 +180,11 @@ func TestFlags(t *testing.T) {
for _, subTest := range subTests { for _, subTest := range subTests {
t.Run(subTest.name+" "+subTest.flagKey, func(t *testing.T) { t.Run(subTest.name+" "+subTest.flagKey, func(t *testing.T) {
capture := &eventCapture{}
client, _ := NewWithConfig("Csyjlnlun3OzyNJAafdlv", Config{ client, _ := NewWithConfig("Csyjlnlun3OzyNJAafdlv", Config{
Endpoint: server.URL, Endpoint: server.URL,
BatchSize: 1, // Send immediately
Callback: capture,
}) })
defer client.Close() defer client.Close()
@@ -151,7 +200,8 @@ func TestFlags(t *testing.T) {
t.Errorf("Expected GetFeatureFlagPayload to return %v but was %v", subTest.expected, payload) t.Errorf("Expected GetFeatureFlagPayload to return %v but was %v", subTest.expected, payload)
} }
validateCapturedEvent(t, client) event := capture.waitForEvent(time.Second)
validateCapturedEvent(t, event)
}) })
} }
}) })
@@ -171,7 +221,8 @@ func TestFeatureFlagErrorOnCapturedEvents(t *testing.T) {
})) }))
defer server.Close() defer server.Close()
client, _ := NewWithConfig("test-api-key", Config{Endpoint: server.URL}) capture := &eventCapture{}
client, _ := NewWithConfig("test-api-key", Config{Endpoint: server.URL, Callback: capture, BatchSize: 1})
defer client.Close() defer client.Close()
_, _ = client.GetFeatureFlag(FeatureFlagPayload{ _, _ = client.GetFeatureFlag(FeatureFlagPayload{
@@ -179,7 +230,7 @@ func TestFeatureFlagErrorOnCapturedEvents(t *testing.T) {
DistinctId: "user-123", DistinctId: "user-123",
}) })
event := client.GetLastCapturedEvent() event := capture.waitForEvent(time.Second)
if event == nil { if event == nil {
t.Fatal("Expected a captured event") t.Fatal("Expected a captured event")
} }
@@ -200,7 +251,8 @@ func TestFeatureFlagErrorOnCapturedEvents(t *testing.T) {
})) }))
defer server.Close() defer server.Close()
client, _ := NewWithConfig("test-api-key", Config{Endpoint: server.URL}) capture := &eventCapture{}
client, _ := NewWithConfig("test-api-key", Config{Endpoint: server.URL, Callback: capture, BatchSize: 1})
defer client.Close() defer client.Close()
_, _ = client.GetFeatureFlag(FeatureFlagPayload{ _, _ = client.GetFeatureFlag(FeatureFlagPayload{
@@ -208,7 +260,7 @@ func TestFeatureFlagErrorOnCapturedEvents(t *testing.T) {
DistinctId: "user-123", DistinctId: "user-123",
}) })
event := client.GetLastCapturedEvent() event := capture.waitForEvent(time.Second)
if event == nil { if event == nil {
t.Fatal("Expected a captured event") t.Fatal("Expected a captured event")
} }
@@ -228,7 +280,8 @@ func TestFeatureFlagErrorOnCapturedEvents(t *testing.T) {
})) }))
defer server.Close() defer server.Close()
client, _ := NewWithConfig("test-api-key", Config{Endpoint: server.URL}) capture := &eventCapture{}
client, _ := NewWithConfig("test-api-key", Config{Endpoint: server.URL, Callback: capture, BatchSize: 1})
defer client.Close() defer client.Close()
_, _ = client.GetFeatureFlag(FeatureFlagPayload{ _, _ = client.GetFeatureFlag(FeatureFlagPayload{
@@ -236,7 +289,7 @@ func TestFeatureFlagErrorOnCapturedEvents(t *testing.T) {
DistinctId: "user-123", DistinctId: "user-123",
}) })
event := client.GetLastCapturedEvent() event := capture.waitForEvent(time.Second)
if event == nil { if event == nil {
t.Fatal("Expected a captured event") t.Fatal("Expected a captured event")
} }
@@ -257,7 +310,8 @@ func TestFeatureFlagErrorOnCapturedEvents(t *testing.T) {
})) }))
defer server.Close() defer server.Close()
client, _ := NewWithConfig("test-api-key", Config{Endpoint: server.URL}) capture := &eventCapture{}
client, _ := NewWithConfig("test-api-key", Config{Endpoint: server.URL, Callback: capture, BatchSize: 1})
defer client.Close() defer client.Close()
_, _ = client.GetFeatureFlag(FeatureFlagPayload{ _, _ = client.GetFeatureFlag(FeatureFlagPayload{
@@ -265,7 +319,7 @@ func TestFeatureFlagErrorOnCapturedEvents(t *testing.T) {
DistinctId: "user-123", DistinctId: "user-123",
}) })
event := client.GetLastCapturedEvent() event := capture.waitForEvent(time.Second)
if event == nil { if event == nil {
t.Fatal("Expected a captured event") t.Fatal("Expected a captured event")
} }
@@ -277,12 +331,16 @@ func TestFeatureFlagErrorOnCapturedEvents(t *testing.T) {
t.Run("API error sets $feature_flag_error with status code", func(t *testing.T) { t.Run("API error sets $feature_flag_error with status code", func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasPrefix(r.URL.Path, "/flags") {
w.WriteHeader(http.StatusInternalServerError) w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(`{"error": "Internal Server Error"}`)) w.Write([]byte(`{"error": "Internal Server Error"}`))
}
// /batch/ returns 200 OK (default) to allow event capture
})) }))
defer server.Close() defer server.Close()
client, _ := NewWithConfig("test-api-key", Config{Endpoint: server.URL}) capture := &eventCapture{}
client, _ := NewWithConfig("test-api-key", Config{Endpoint: server.URL, Callback: capture, BatchSize: 1})
defer client.Close() defer client.Close()
_, _ = client.GetFeatureFlag(FeatureFlagPayload{ _, _ = client.GetFeatureFlag(FeatureFlagPayload{
@@ -290,7 +348,7 @@ func TestFeatureFlagErrorOnCapturedEvents(t *testing.T) {
DistinctId: "user-123", DistinctId: "user-123",
}) })
event := client.GetLastCapturedEvent() event := capture.waitForEvent(time.Second)
if event == nil { if event == nil {
t.Fatal("Expected a captured event") t.Fatal("Expected a captured event")
} }
@@ -310,7 +368,8 @@ func TestFeatureFlagErrorOnCapturedEvents(t *testing.T) {
})) }))
defer server.Close() defer server.Close()
client, _ := NewWithConfig("test-api-key", Config{Endpoint: server.URL}) capture := &eventCapture{}
client, _ := NewWithConfig("test-api-key", Config{Endpoint: server.URL, Callback: capture, BatchSize: 1})
defer client.Close() defer client.Close()
_, _ = client.GetFeatureFlag(FeatureFlagPayload{ _, _ = client.GetFeatureFlag(FeatureFlagPayload{
@@ -318,7 +377,7 @@ func TestFeatureFlagErrorOnCapturedEvents(t *testing.T) {
DistinctId: "user-123", DistinctId: "user-123",
}) })
event := client.GetLastCapturedEvent() event := capture.waitForEvent(time.Second)
if event == nil { if event == nil {
t.Fatal("Expected a captured event") t.Fatal("Expected a captured event")
} }
@@ -337,11 +396,15 @@ func TestFeatureFlagErrorOnCapturedEvents(t *testing.T) {
t.Run("$feature_flag_errored is true when errors exist", func(t *testing.T) { t.Run("$feature_flag_errored is true when errors exist", func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasPrefix(r.URL.Path, "/flags") {
w.WriteHeader(http.StatusInternalServerError) w.WriteHeader(http.StatusInternalServerError)
}
// /batch/ returns 200 OK (default) to allow event capture
})) }))
defer server.Close() defer server.Close()
client, _ := NewWithConfig("test-api-key", Config{Endpoint: server.URL}) capture := &eventCapture{}
client, _ := NewWithConfig("test-api-key", Config{Endpoint: server.URL, Callback: capture, BatchSize: 1})
defer client.Close() defer client.Close()
_, err := client.GetFeatureFlag(FeatureFlagPayload{ _, err := client.GetFeatureFlag(FeatureFlagPayload{
@@ -353,7 +416,7 @@ func TestFeatureFlagErrorOnCapturedEvents(t *testing.T) {
t.Error("Expected an error from GetFeatureFlag") t.Error("Expected an error from GetFeatureFlag")
} }
event := client.GetLastCapturedEvent() event := capture.waitForEvent(time.Second)
if event == nil { if event == nil {
t.Fatal("Expected a captured event") t.Fatal("Expected a captured event")
} }
@@ -371,11 +434,6 @@ func TestFlagsV4(t *testing.T) {
})) }))
defer server.Close() defer server.Close()
client, _ := NewWithConfig("Csyjlnlun3OzyNJAafdlv", Config{
Endpoint: server.URL,
})
defer client.Close()
tests := []struct { tests := []struct {
name string name string
flagKey string flagKey string
@@ -390,7 +448,15 @@ func TestFlagsV4(t *testing.T) {
} }
for _, test := range tests { for _, test := range tests {
t.Run(test.name, func(t *testing.T) { t.Run(test.name+" "+test.flagKey, func(t *testing.T) {
capture := &eventCapture{}
client, _ := NewWithConfig("Csyjlnlun3OzyNJAafdlv", Config{
Endpoint: server.URL,
BatchSize: 1, // Send immediately
Callback: capture,
})
defer client.Close()
flag, _ := client.GetFeatureFlag( flag, _ := client.GetFeatureFlag(
FeatureFlagPayload{ FeatureFlagPayload{
Key: test.flagKey, Key: test.flagKey,
@@ -402,9 +468,9 @@ func TestFlagsV4(t *testing.T) {
t.Errorf("Expected GetFeatureFlag(%s) to return %v but was %v", test.flagKey, test.expected, flag) t.Errorf("Expected GetFeatureFlag(%s) to return %v but was %v", test.flagKey, test.expected, flag)
} }
capturedEvent := client.GetLastCapturedEvent() capturedEvent := capture.waitForEvent(time.Second)
if capturedEvent == nil { if capturedEvent == nil {
t.Errorf("Expected a $feature_flag_called for %s event, got: %v", test.flagKey, capturedEvent) t.Fatalf("Expected a $feature_flag_called for %s event, got nil", test.flagKey)
} }
if capturedEvent.Properties["$feature_flag"] != test.flagKey { if capturedEvent.Properties["$feature_flag"] != test.flagKey {
File diff suppressed because it is too large Load Diff
+559
View File
@@ -0,0 +1,559 @@
package posthog
import (
"errors"
"testing"
"time"
"github.com/stretchr/testify/require"
)
func TestMatchPropertyValue(t *testing.T) {
property := FlagProperty{
Key: "Browser",
Value: "Chrome",
Operator: "exact",
}
properties := NewProperties().Set("Browser", "Chrome")
isMatch, err := matchProperty(property, properties)
if err != nil || !isMatch {
t.Error("Value is not a match")
}
}
func TestMatchPropertyInvalidOperator(t *testing.T) {
property := FlagProperty{
Key: "Browser",
Value: "Chrome",
Operator: "is_unknown",
}
properties := NewProperties().Set("Browser", "Chrome")
isMatch, err := matchProperty(property, properties)
if isMatch == true {
t.Error("Should not match")
}
var inconclusiveErr *InconclusiveMatchError
if !errors.As(err, &inconclusiveErr) {
t.Error("Error type is not a match")
}
}
func TestMatchPropertySlice(t *testing.T) {
property := FlagProperty{
Key: "Browser",
Value: []interface{}{"Chrome", "Firefox"},
Operator: "exact",
}
for _, tt := range []struct {
name string
properties Properties
expected bool
err error
}{
{
name: "match with Chrome",
properties: NewProperties().Set("Browser", "Chrome"),
expected: true,
},
{
name: "match with Firefox",
properties: NewProperties().Set("Browser", "Firefox"),
expected: true,
},
{
name: "no match with Explorer",
properties: NewProperties().Set("Browser", "Explorer"),
},
{
name: "no match with unknown key",
properties: NewProperties().Set("Car", "Chrome"),
err: errors.New(""),
},
} {
t.Run(tt.name, func(t *testing.T) {
isMatch, err := matchProperty(property, tt.properties)
if tt.err != nil {
require.Error(t, err)
} else {
require.NoError(t, err)
}
require.Equal(t, tt.expected, isMatch)
})
}
}
func TestMatchPropertySliceExact(t *testing.T) {
property := FlagProperty{
Key: "Browser",
Value: []interface{}{"Chrome", "Firefox"},
Operator: "exact",
}
isMatch, err := matchProperty(property, NewProperties().Set("Browser", "Chrome"))
require.NoError(t, err)
require.True(t, isMatch)
isMatch, err = matchProperty(property, NewProperties().Set("Browser", "Firefox"))
require.NoError(t, err)
require.True(t, isMatch)
isMatch, err = matchProperty(property, NewProperties().Set("Browser", "Explorer"))
require.NoError(t, err)
require.False(t, isMatch)
isMatch, err = matchProperty(property, NewProperties().Set("Car", "Fiat"))
require.Error(t, err)
require.False(t, isMatch)
}
func TestMatchPropertyNumber(t *testing.T) {
property := FlagProperty{
Key: "Number",
Value: 5,
Operator: "gt",
}
properties := NewProperties().Set("Number", 7)
isMatch, err := matchProperty(property, properties)
if err != nil {
t.Error(err)
}
if !isMatch {
t.Error("Value is not a match")
}
property = FlagProperty{
Key: "Number",
Value: 5,
Operator: "lt",
}
properties = NewProperties().Set("Number", 4)
isMatch, err = matchProperty(property, properties)
if err != nil {
t.Error(err)
}
if !isMatch {
t.Error("Value is not a match")
}
property = FlagProperty{
Key: "Number",
Value: 5,
Operator: "gte",
}
properties = NewProperties().Set("Number", 5)
isMatch, err = matchProperty(property, properties)
if err != nil {
t.Error(err)
}
if !isMatch {
t.Error("Value is not a match")
}
property = FlagProperty{
Key: "Number",
Value: 5,
Operator: "lte",
}
properties = NewProperties().Set("Number", 4)
isMatch, err = matchProperty(property, properties)
if err != nil {
t.Error(err)
}
if !isMatch {
t.Error("Value is not a match")
}
}
func TestMatchPropertyRegex(t *testing.T) {
shouldMatch := []interface{}{"value.com", "value2.com"}
property := FlagProperty{
Key: "key",
Value: "\\.com$",
Operator: "regex",
}
for _, val := range shouldMatch {
isMatch, err := matchProperty(property, NewProperties().Set("key", val))
if err != nil {
t.Error(err)
}
if !isMatch {
t.Error("Value is not a match")
}
}
shouldNotMatch := []interface{}{".com343tfvalue5", "Alakazam", 123}
for _, val := range shouldNotMatch {
isMatch, err := matchProperty(property, NewProperties().Set("key", val))
if err != nil {
t.Error(err)
}
if isMatch {
t.Error("Value is not a match")
}
}
// invalid regex
property = FlagProperty{
Key: "key",
Value: "?*",
Operator: "regex",
}
shouldNotMatch = []interface{}{"value", "valu2"}
for _, val := range shouldNotMatch {
isMatch, err := matchProperty(property, NewProperties().Set("key", val))
if err != nil {
t.Error(err)
}
if isMatch {
t.Error("Value is not a match")
}
}
// non string value
property = FlagProperty{
Key: "key",
Value: 4,
Operator: "regex",
}
shouldMatch = []interface{}{"4", 4}
for _, val := range shouldMatch {
isMatch, err := matchProperty(property, NewProperties().Set("key", val))
if err != nil {
t.Error(err)
}
if !isMatch {
t.Error("Value is not a match")
}
}
}
func TestMatchPropertyContains(t *testing.T) {
shouldMatch := []interface{}{"value", "value2", "value3", "value4", "343tfvalue5"}
property := FlagProperty{
Key: "key",
Value: "valUe",
Operator: "icontains",
}
for _, val := range shouldMatch {
isMatch, err := matchProperty(property, NewProperties().Set("key", val))
if err != nil {
t.Error(err)
}
if !isMatch {
t.Error("Value is not a match")
}
}
shouldNotMatch := []interface{}{"Alakazam", 123}
for _, val := range shouldNotMatch {
isMatch, err := matchProperty(property, NewProperties().Set("key", val))
if err != nil {
t.Error(err)
}
if isMatch {
t.Error("Value is not a match")
}
}
}
func TestMatchPropertyDateComparison(t *testing.T) {
t.Run("RFC3339 dates", func(t *testing.T) {
// Test is_date_before
property := FlagProperty{
Key: "created_at",
Value: "2024-12-31T23:59:59Z",
Operator: "is_date_before",
}
// Should match: 2024-06-15 is before 2024-12-31
isMatch, err := matchProperty(property, NewProperties().Set("created_at", "2024-06-15T10:00:00Z"))
require.NoError(t, err)
require.True(t, isMatch)
// Should not match: 2025-01-01 is after 2024-12-31
isMatch, err = matchProperty(property, NewProperties().Set("created_at", "2025-01-01T00:00:00Z"))
require.NoError(t, err)
require.False(t, isMatch)
// Test is_date_after
property.Operator = "is_date_after"
// Should match: 2025-01-01 is after 2024-12-31
isMatch, err = matchProperty(property, NewProperties().Set("created_at", "2025-01-01T00:00:00Z"))
require.NoError(t, err)
require.True(t, isMatch)
// Should not match: 2024-06-15 is before 2024-12-31
isMatch, err = matchProperty(property, NewProperties().Set("created_at", "2024-06-15T10:00:00Z"))
require.NoError(t, err)
require.False(t, isMatch)
})
t.Run("ISO 8601 date formats", func(t *testing.T) {
// Test date-only format (YYYY-MM-DD)
property := FlagProperty{
Key: "created_at",
Value: "2024-12-31",
Operator: "is_date_before",
}
// Should match: 2024-06-15 is before 2024-12-31
isMatch, err := matchProperty(property, NewProperties().Set("created_at", "2024-06-15"))
require.NoError(t, err)
require.True(t, isMatch)
// Should not match: 2025-01-01 is after 2024-12-31
isMatch, err = matchProperty(property, NewProperties().Set("created_at", "2025-01-01"))
require.NoError(t, err)
require.False(t, isMatch)
// Test datetime without timezone (YYYY-MM-DDTHH:MM:SS)
property.Value = "2024-12-31T23:59:59"
// Should match: 2024-06-15T10:00:00 is before 2024-12-31T23:59:59
isMatch, err = matchProperty(property, NewProperties().Set("created_at", "2024-06-15T10:00:00"))
require.NoError(t, err)
require.True(t, isMatch)
// Should not match: 2025-01-01T00:00:00 is after 2024-12-31T23:59:59
isMatch, err = matchProperty(property, NewProperties().Set("created_at", "2025-01-01T00:00:00"))
require.NoError(t, err)
require.False(t, isMatch)
// Test is_date_after with date-only format
property.Operator = "is_date_after"
property.Value = "2024-06-15"
// Should match: 2025-01-01 is after 2024-06-15
isMatch, err = matchProperty(property, NewProperties().Set("created_at", "2025-01-01"))
require.NoError(t, err)
require.True(t, isMatch)
// Should not match: 2024-01-01 is before 2024-06-15
isMatch, err = matchProperty(property, NewProperties().Set("created_at", "2024-01-01"))
require.NoError(t, err)
require.False(t, isMatch)
// Test mixing formats (RFC3339 value with date-only property)
property.Value = "2024-06-15"
isMatch, err = matchProperty(property, NewProperties().Set("created_at", "2025-01-01T00:00:00Z"))
require.NoError(t, err)
require.True(t, isMatch)
// Test mixing formats (date-only value with datetime property)
property.Value = "2024-06-15T12:00:00"
isMatch, err = matchProperty(property, NewProperties().Set("created_at", "2025-01-01"))
require.NoError(t, err)
require.True(t, isMatch)
})
t.Run("ISO 8601 fractional seconds and timezone offsets", func(t *testing.T) {
// Test fractional seconds with Z timezone
property := FlagProperty{
Key: "created_at",
Value: "2024-12-31T23:59:59.999Z",
Operator: "is_date_before",
}
// Should match: earlier fractional time
isMatch, err := matchProperty(property, NewProperties().Set("created_at", "2024-06-15T10:30:00.123Z"))
require.NoError(t, err)
require.True(t, isMatch)
// Should not match: later time
isMatch, err = matchProperty(property, NewProperties().Set("created_at", "2025-01-01T00:00:00.001Z"))
require.NoError(t, err)
require.False(t, isMatch)
// Test timezone offsets (positive)
property.Value = "2024-06-15T10:00:00+05:30"
// Should match: earlier time with different timezone
isMatch, err = matchProperty(property, NewProperties().Set("created_at", "2024-06-15T03:00:00Z"))
require.NoError(t, err)
require.True(t, isMatch)
// Should not match: later time
isMatch, err = matchProperty(property, NewProperties().Set("created_at", "2024-06-15T12:00:00+05:30"))
require.NoError(t, err)
require.False(t, isMatch)
// Test timezone offsets (negative)
property.Value = "2024-06-15T10:00:00-08:00"
// Should match: earlier time
isMatch, err = matchProperty(property, NewProperties().Set("created_at", "2024-06-15T08:00:00-08:00"))
require.NoError(t, err)
require.True(t, isMatch)
// Test combined fractional seconds + timezone offset
property.Value = "2024-12-31T23:59:59.999+01:00"
// Should match: earlier fractional time with timezone
isMatch, err = matchProperty(property, NewProperties().Set("created_at", "2024-12-31T22:59:59.998+01:00"))
require.NoError(t, err)
require.True(t, isMatch)
// Should not match: later time
isMatch, err = matchProperty(property, NewProperties().Set("created_at", "2025-01-01T00:00:00.000+01:00"))
require.NoError(t, err)
require.False(t, isMatch)
// Test various fractional second precisions
testCases := []struct {
name string
dateStr string
expected bool
}{
{"1 digit fractional", "2024-01-15T10:30:00.1Z", true},
{"2 digit fractional", "2024-01-15T10:30:00.12Z", true},
{"3 digit fractional", "2024-01-15T10:30:00.123Z", true},
{"6 digit fractional", "2024-01-15T10:30:00.123456Z", true},
{"9 digit fractional", "2024-01-15T10:30:00.123456789Z", true},
{"fractional with +00:00", "2024-01-15T10:30:00.123+00:00", true},
}
property.Operator = "is_date_after"
property.Value = "2024-01-01T00:00:00Z"
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
isMatch, err := matchProperty(property, NewProperties().Set("created_at", tc.dateStr))
require.NoError(t, err)
require.Equal(t, tc.expected, isMatch, "Failed for date: %s", tc.dateStr)
})
}
// Test mixing fractional seconds with non-fractional
property.Value = "2024-06-15T10:30:00Z"
isMatch, err = matchProperty(property, NewProperties().Set("created_at", "2024-06-15T10:30:00.999Z"))
require.NoError(t, err)
require.True(t, isMatch) // .999 is after :00 (operator is still is_date_after)
// Test +00:00 equivalent to Z
property.Value = "2024-06-15T10:30:00+00:00"
isMatch, err = matchProperty(property, NewProperties().Set("created_at", "2024-06-15T10:30:00Z"))
require.NoError(t, err)
require.False(t, isMatch) // Equal times, not after
})
t.Run("Relative dates", func(t *testing.T) {
now := time.Now()
// Test is_date_after with relative date
property := FlagProperty{
Key: "created_at",
Value: "-7d", // 7 days ago
Operator: "is_date_after",
}
// Should match: 3 days ago is after 7 days ago
threeDaysAgo := now.AddDate(0, 0, -3).Format(time.RFC3339)
isMatch, err := matchProperty(property, NewProperties().Set("created_at", threeDaysAgo))
require.NoError(t, err)
require.True(t, isMatch)
// Should not match: 10 days ago is before 7 days ago
tenDaysAgo := now.AddDate(0, 0, -10).Format(time.RFC3339)
isMatch, err = matchProperty(property, NewProperties().Set("created_at", tenDaysAgo))
require.NoError(t, err)
require.False(t, isMatch)
// Test is_date_before with relative date
property.Operator = "is_date_before"
// Should match: 10 days ago is before 7 days ago
isMatch, err = matchProperty(property, NewProperties().Set("created_at", tenDaysAgo))
require.NoError(t, err)
require.True(t, isMatch)
})
t.Run("Various relative date formats", func(t *testing.T) {
testCases := []struct {
name string
relativeDate string
shouldParse bool
}{
{"1 hour", "1h", true},
{"7 days", "7d", true},
{"2 weeks", "2w", true},
{"3 months", "3m", true},
{"1 year", "1y", true},
{"large number rejected", "10000d", false},
{"invalid format", "invalid", false},
{"no number", "d", false},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
_, err := parseRelativeDate(tc.relativeDate)
if tc.shouldParse {
require.NoError(t, err)
} else {
require.Error(t, err)
}
})
}
})
t.Run("Error handling", func(t *testing.T) {
property := FlagProperty{
Key: "created_at",
Value: "2024-12-31T23:59:59Z",
Operator: "is_date_before",
}
// Test with non-string value
isMatch, err := matchProperty(property, NewProperties().Set("created_at", 12345))
require.Error(t, err)
require.False(t, isMatch)
// Test with invalid date format
isMatch, err = matchProperty(property, NewProperties().Set("created_at", "invalid-date"))
require.Error(t, err)
require.False(t, isMatch)
})
}
+83 -38
View File
@@ -5,16 +5,17 @@ import (
"context" "context"
"crypto/sha1" "crypto/sha1"
"encoding/binary" "encoding/binary"
"encoding/json"
"errors" "errors"
"fmt" "fmt"
json "github.com/goccy/go-json"
"io" "io"
"net/http" "net/http"
"net/url" "net/url"
"regexp" "regexp"
"strconv" "strconv"
"strings" "strings"
"sync" "sync/atomic"
"time" "time"
) )
@@ -22,27 +23,35 @@ const LONG_SCALE = 0xfffffffffffffff
var relativeDateRegex = regexp.MustCompile(`^-?([0-9]+)([hdwmy])$`) var relativeDateRegex = regexp.MustCompile(`^-?([0-9]+)([hdwmy])$`)
// flagsState holds the feature flag data that is atomically swapped during updates.
// This provides lock-free reads for the common path (flag evaluation).
type flagsState struct {
featureFlags []FeatureFlag
cohorts map[string]PropertyGroup
groups map[string]string
flagsEtag string
}
type FeatureFlagsPoller struct { type FeatureFlagsPoller struct {
// firstFeatureFlagRequestFinished is used to log feature flag usage before the first feature flag request is done. // firstFeatureFlagRequestFinished is used to log feature flag usage before the first feature flag request is done.
// After the request the channel get closed. // After the request the channel get closed.
firstFeatureFlagRequestFinished chan bool firstFeatureFlagRequestFinished chan bool
shutdown chan bool shutdown chan bool
forceReload chan bool forceReload chan bool
featureFlags []FeatureFlag
cohorts map[string]PropertyGroup // state holds all flag-related data using atomic pointer for lock-free reads
groups map[string]string state atomic.Pointer[flagsState]
personalApiKey string personalApiKey string
projectApiKey string projectApiKey string
localEvalUrl *url.URL localEvalUrl *url.URL
Logger Logger Logger Logger
Endpoint string Endpoint string
http http.Client http http.Client
mutex sync.RWMutex
nextPollTick func() time.Duration nextPollTick func() time.Duration
flagTimeout time.Duration flagTimeout time.Duration
decider decider decider decider
disableGeoIP bool disableGeoIP bool
flagsEtag string // ETag for conditional requests to reduce bandwidth
} }
type FeatureFlag struct { type FeatureFlag struct {
@@ -291,7 +300,6 @@ func newFeatureFlagsPoller(
Logger: logger, Logger: logger,
Endpoint: endpoint, Endpoint: endpoint,
http: httpClient, http: httpClient,
mutex: sync.RWMutex{},
nextPollTick: nextPollTick, nextPollTick: nextPollTick,
flagTimeout: flagTimeout, flagTimeout: flagTimeout,
decider: decider, decider: decider,
@@ -329,10 +337,12 @@ func (poller *FeatureFlagsPoller) fetchNewFeatureFlags() {
personalApiKey := poller.personalApiKey personalApiKey := poller.personalApiKey
headers := http.Header{"Authorization": []string{"Bearer " + personalApiKey}} headers := http.Header{"Authorization": []string{"Bearer " + personalApiKey}}
// Read current ETag under lock // Read current ETag from state (lock-free)
poller.mutex.RLock() currentState := poller.state.Load()
currentEtag := poller.flagsEtag currentEtag := ""
poller.mutex.RUnlock() if currentState != nil {
currentEtag = currentState.flagsEtag
}
res, cancel, err := poller.localEvaluationFlags(headers, currentEtag) res, cancel, err := poller.localEvaluationFlags(headers, currentEtag)
if err != nil { if err != nil {
@@ -346,23 +356,28 @@ func (poller *FeatureFlagsPoller) fetchNewFeatureFlags() {
if res.StatusCode == http.StatusNotModified { if res.StatusCode == http.StatusNotModified {
poller.Logger.Debugf("[FEATURE FLAGS] Flags not modified (304), using cached data") poller.Logger.Debugf("[FEATURE FLAGS] Flags not modified (304), using cached data")
// Update ETag if server returned one (preserve existing if not) // Update ETag if server returned one (preserve existing if not)
if newEtag := res.Header.Get("ETag"); newEtag != "" { if newEtag := res.Header.Get("ETag"); newEtag != "" && currentState != nil {
poller.mutex.Lock() // Atomically swap with updated ETag
poller.flagsEtag = newEtag newState := &flagsState{
poller.mutex.Unlock() featureFlags: currentState.featureFlags,
cohorts: currentState.cohorts,
groups: currentState.groups,
flagsEtag: newEtag,
}
poller.state.Store(newState)
} }
return return
} }
// Handle quota limit response (HTTP 402) // Handle quota limit response (HTTP 402)
if res.StatusCode == http.StatusPaymentRequired { if res.StatusCode == http.StatusPaymentRequired {
// Clear existing flags when quota limited // Clear existing flags when quota limited - atomic swap
poller.mutex.Lock() poller.state.Store(&flagsState{
poller.featureFlags = []FeatureFlag{} featureFlags: []FeatureFlag{},
poller.cohorts = map[string]PropertyGroup{} cohorts: map[string]PropertyGroup{},
poller.groups = map[string]string{} groups: map[string]string{},
poller.flagsEtag = "" // Clear ETag on quota limit flagsEtag: "",
poller.mutex.Unlock() })
poller.Logger.Warnf("[FEATURE FLAGS] PostHog feature flags quota limited, resetting feature flag data. Learn more about billing limits at https://posthog.com/docs/billing/limits-alerts") poller.Logger.Warnf("[FEATURE FLAGS] PostHog feature flags quota limited, resetting feature flag data. Learn more about billing limits at https://posthog.com/docs/billing/limits-alerts")
return return
} }
@@ -387,14 +402,19 @@ func (poller *FeatureFlagsPoller) fetchNewFeatureFlags() {
// Store new ETag from response (clear if server stops sending) // Store new ETag from response (clear if server stops sending)
newEtag := res.Header.Get("ETag") newEtag := res.Header.Get("ETag")
poller.mutex.Lock() // Build new groups map
defer poller.mutex.Unlock() groups := map[string]string{}
poller.featureFlags = newFlags
poller.cohorts = featureFlagsResponse.Cohorts
poller.flagsEtag = newEtag
if featureFlagsResponse.GroupTypeMapping != nil { if featureFlagsResponse.GroupTypeMapping != nil {
poller.groups = *featureFlagsResponse.GroupTypeMapping groups = *featureFlagsResponse.GroupTypeMapping
} }
// Atomic swap of entire state
poller.state.Store(&flagsState{
featureFlags: newFlags,
cohorts: featureFlagsResponse.Cohorts,
groups: groups,
flagsEtag: newEtag,
})
} }
func (poller *FeatureFlagsPoller) GetFeatureFlag(flagConfig FeatureFlagPayload) (interface{}, error) { func (poller *FeatureFlagsPoller) GetFeatureFlag(flagConfig FeatureFlagPayload) (interface{}, error) {
@@ -409,7 +429,7 @@ func (poller *FeatureFlagsPoller) GetFeatureFlag(flagConfig FeatureFlagPayload)
flagConfig.Groups, flagConfig.Groups,
flagConfig.PersonProperties, flagConfig.PersonProperties,
flagConfig.GroupProperties, flagConfig.GroupProperties,
poller.cohorts, poller.getCohorts(),
) )
} }
@@ -439,7 +459,7 @@ func (poller *FeatureFlagsPoller) GetFeatureFlagPayload(flagConfig FeatureFlagPa
flagConfig.Groups, flagConfig.Groups,
flagConfig.PersonProperties, flagConfig.PersonProperties,
flagConfig.GroupProperties, flagConfig.GroupProperties,
poller.cohorts, poller.getCohorts(),
) )
} }
if err != nil { if err != nil {
@@ -488,7 +508,7 @@ func (poller *FeatureFlagsPoller) GetAllFlags(flagConfig FeatureFlagPayloadNoKey
return nil, err return nil, err
} }
fallbackToDecide := false fallbackToDecide := false
cohorts := poller.cohorts cohorts := poller.getCohorts()
if len(featureFlags) == 0 { if len(featureFlags) == 0 {
fallbackToDecide = true fallbackToDecide = true
@@ -561,7 +581,7 @@ func (poller *FeatureFlagsPoller) computeFlagLocally(
} }
if flag.Filters.AggregationGroupTypeIndex != nil { if flag.Filters.AggregationGroupTypeIndex != nil {
groupType, exists := poller.groups[fmt.Sprintf("%d", *flag.Filters.AggregationGroupTypeIndex)] groupType, exists := poller.getGroups()[fmt.Sprintf("%d", *flag.Filters.AggregationGroupTypeIndex)]
if !exists { if !exists {
errMessage := "flag has unknown group type index" errMessage := "flag has unknown group type index"
@@ -1149,14 +1169,39 @@ func calculateHash(prefix, distinctId, salt string) float64 {
func (poller *FeatureFlagsPoller) GetFeatureFlags() ([]FeatureFlag, error) { func (poller *FeatureFlagsPoller) GetFeatureFlags() ([]FeatureFlag, error) {
// When channel is open this will block. When channel is closed it will immediately exit. // When channel is open this will block. When channel is closed it will immediately exit.
<-poller.firstFeatureFlagRequestFinished <-poller.firstFeatureFlagRequestFinished
poller.mutex.RLock()
defer poller.mutex.RUnlock() // Lock-free read of state
if poller.featureFlags == nil { state := poller.state.Load()
if state == nil || state.featureFlags == nil {
// There was an error with initial flag fetching // There was an error with initial flag fetching
return nil, errors.New("flags were not successfully fetched yet") return nil, errors.New("flags were not successfully fetched yet")
} }
return poller.featureFlags, nil return state.featureFlags, nil
}
// getState returns the current flags state or nil if not initialized.
// This is a lock-free read.
func (poller *FeatureFlagsPoller) getState() *flagsState {
return poller.state.Load()
}
// getCohorts returns the current cohorts map or an empty map if not initialized.
func (poller *FeatureFlagsPoller) getCohorts() map[string]PropertyGroup {
state := poller.state.Load()
if state == nil || state.cohorts == nil {
return map[string]PropertyGroup{}
}
return state.cohorts
}
// getGroups returns the current groups map or an empty map if not initialized.
func (poller *FeatureFlagsPoller) getGroups() map[string]string {
state := poller.state.Load()
if state == nil || state.groups == nil {
return map[string]string{}
}
return state.groups
} }
func (poller *FeatureFlagsPoller) localEvaluationFlags(headers http.Header, etag string) (*http.Response, context.CancelFunc, error) { func (poller *FeatureFlagsPoller) localEvaluationFlags(headers http.Header, etag string) (*http.Response, context.CancelFunc, error) {
@@ -1225,7 +1270,7 @@ func (poller *FeatureFlagsPoller) getFeatureFlagVariantsLocalOnly(distinctId str
} }
result := make(map[string]interface{}) result := make(map[string]interface{})
cohorts := poller.cohorts cohorts := poller.getCohorts()
for _, flag := range flags { for _, flag := range flags {
flagValue, err := poller.computeFlagLocally( flagValue, err := poller.computeFlagLocally(
+57 -44
View File
@@ -5,7 +5,6 @@ import (
"net/http/httptest" "net/http/httptest"
"net/url" "net/url"
"strings" "strings"
"sync"
"testing" "testing"
"time" "time"
) )
@@ -27,7 +26,6 @@ func newTestPoller(t *testing.T, serverURL string) *FeatureFlagsPoller {
Logger: newDefaultLogger(false), Logger: newDefaultLogger(false),
Endpoint: serverURL, Endpoint: serverURL,
http: http.Client{}, http: http.Client{},
mutex: sync.RWMutex{},
flagTimeout: 10 * time.Second, flagTimeout: 10 * time.Second,
} }
} }
@@ -49,9 +47,11 @@ func TestETagSupportForLocalEvaluation(t *testing.T) {
poller := newTestPoller(t, server.URL) poller := newTestPoller(t, server.URL)
poller.fetchNewFeatureFlags() poller.fetchNewFeatureFlags()
poller.mutex.RLock() state := poller.state.Load()
etag := poller.flagsEtag etag := ""
poller.mutex.RUnlock() if state != nil {
etag = state.flagsEtag
}
if etag != `"abc123"` { if etag != `"abc123"` {
t.Errorf("Expected ETag to be \"abc123\", got %q", etag) t.Errorf("Expected ETag to be \"abc123\", got %q", etag)
@@ -128,24 +128,18 @@ func TestETagSupportForLocalEvaluation(t *testing.T) {
// First request - get flags // First request - get flags
poller.fetchNewFeatureFlags() poller.fetchNewFeatureFlags()
poller.mutex.RLock() state := poller.state.Load()
flags := poller.featureFlags if state == nil || len(state.featureFlags) != 1 || state.featureFlags[0].Key != "cached-flag" {
poller.mutex.RUnlock() t.Errorf("Expected initial flag 'cached-flag', got %+v", state)
if len(flags) != 1 || flags[0].Key != "cached-flag" {
t.Errorf("Expected initial flag 'cached-flag', got %+v", flags)
} }
// Second request - 304 // Second request - 304
poller.fetchNewFeatureFlags() poller.fetchNewFeatureFlags()
// Flags should still be available after 304 // Flags should still be available after 304
poller.mutex.RLock() state = poller.state.Load()
flags = poller.featureFlags if state == nil || len(state.featureFlags) != 1 || state.featureFlags[0].Key != "cached-flag" {
poller.mutex.RUnlock() t.Errorf("Expected flags to remain 'cached-flag' after 304, got %+v", state)
if len(flags) != 1 || flags[0].Key != "cached-flag" {
t.Errorf("Expected flags to remain 'cached-flag' after 304, got %+v", flags)
} }
}) })
@@ -179,9 +173,11 @@ func TestETagSupportForLocalEvaluation(t *testing.T) {
poller.fetchNewFeatureFlags() poller.fetchNewFeatureFlags()
poller.mutex.RLock() state := poller.state.Load()
etag1 := poller.flagsEtag etag1 := ""
poller.mutex.RUnlock() if state != nil {
etag1 = state.flagsEtag
}
if etag1 != `"etag-v1"` { if etag1 != `"etag-v1"` {
t.Errorf("Expected initial ETag to be \"etag-v1\", got %q", etag1) t.Errorf("Expected initial ETag to be \"etag-v1\", got %q", etag1)
@@ -190,10 +186,13 @@ func TestETagSupportForLocalEvaluation(t *testing.T) {
// Second request - new ETag // Second request - new ETag
poller.fetchNewFeatureFlags() poller.fetchNewFeatureFlags()
poller.mutex.RLock() state = poller.state.Load()
etag2 := poller.flagsEtag etag2 := ""
flags := poller.featureFlags var flags []FeatureFlag
poller.mutex.RUnlock() if state != nil {
etag2 = state.flagsEtag
flags = state.featureFlags
}
if etag2 != `"etag-v2"` { if etag2 != `"etag-v2"` {
t.Errorf("Expected updated ETag to be \"etag-v2\", got %q", etag2) t.Errorf("Expected updated ETag to be \"etag-v2\", got %q", etag2)
@@ -233,9 +232,11 @@ func TestETagSupportForLocalEvaluation(t *testing.T) {
poller.fetchNewFeatureFlags() poller.fetchNewFeatureFlags()
poller.mutex.RLock() state := poller.state.Load()
etag1 := poller.flagsEtag etag1 := ""
poller.mutex.RUnlock() if state != nil {
etag1 = state.flagsEtag
}
if etag1 != `"etag-v1"` { if etag1 != `"etag-v1"` {
t.Errorf("Expected initial ETag to be \"etag-v1\", got %q", etag1) t.Errorf("Expected initial ETag to be \"etag-v1\", got %q", etag1)
@@ -244,9 +245,11 @@ func TestETagSupportForLocalEvaluation(t *testing.T) {
// Second request - no ETag // Second request - no ETag
poller.fetchNewFeatureFlags() poller.fetchNewFeatureFlags()
poller.mutex.RLock() state = poller.state.Load()
etag2 := poller.flagsEtag etag2 := ""
poller.mutex.RUnlock() if state != nil {
etag2 = state.flagsEtag
}
if etag2 != "" { if etag2 != "" {
t.Errorf("Expected ETag to be cleared when server stops sending it, got %q", etag2) t.Errorf("Expected ETag to be cleared when server stops sending it, got %q", etag2)
@@ -279,9 +282,11 @@ func TestETagSupportForLocalEvaluation(t *testing.T) {
poller.fetchNewFeatureFlags() poller.fetchNewFeatureFlags()
poller.mutex.RLock() state := poller.state.Load()
etag1 := poller.flagsEtag etag1 := ""
poller.mutex.RUnlock() if state != nil {
etag1 = state.flagsEtag
}
if etag1 != `"etag-v1"` { if etag1 != `"etag-v1"` {
t.Errorf("Expected initial ETag to be \"etag-v1\", got %q", etag1) t.Errorf("Expected initial ETag to be \"etag-v1\", got %q", etag1)
@@ -290,10 +295,13 @@ func TestETagSupportForLocalEvaluation(t *testing.T) {
// Second request - 304 without ETag // Second request - 304 without ETag
poller.fetchNewFeatureFlags() poller.fetchNewFeatureFlags()
poller.mutex.RLock() state = poller.state.Load()
etag2 := poller.flagsEtag etag2 := ""
flags := poller.featureFlags var flags []FeatureFlag
poller.mutex.RUnlock() if state != nil {
etag2 = state.flagsEtag
flags = state.featureFlags
}
if etag2 != `"etag-v1"` { if etag2 != `"etag-v1"` {
t.Errorf("Expected ETag to be preserved when 304 has no ETag header, got %q", etag2) t.Errorf("Expected ETag to be preserved when 304 has no ETag header, got %q", etag2)
@@ -332,9 +340,11 @@ func TestETagSupportForLocalEvaluation(t *testing.T) {
poller.fetchNewFeatureFlags() poller.fetchNewFeatureFlags()
poller.mutex.RLock() state := poller.state.Load()
etag1 := poller.flagsEtag etag1 := ""
poller.mutex.RUnlock() if state != nil {
etag1 = state.flagsEtag
}
if etag1 != `"etag-v1"` { if etag1 != `"etag-v1"` {
t.Errorf("Expected initial ETag to be \"etag-v1\", got %q", etag1) t.Errorf("Expected initial ETag to be \"etag-v1\", got %q", etag1)
@@ -343,10 +353,13 @@ func TestETagSupportForLocalEvaluation(t *testing.T) {
// Second request - quota limited // Second request - quota limited
poller.fetchNewFeatureFlags() poller.fetchNewFeatureFlags()
poller.mutex.RLock() state = poller.state.Load()
etag2 := poller.flagsEtag etag2 := ""
flags := poller.featureFlags var flags []FeatureFlag
poller.mutex.RUnlock() if state != nil {
etag2 = state.flagsEtag
flags = state.featureFlags
}
if etag2 != "" { if etag2 != "" {
t.Errorf("Expected ETag to be cleared on quota limit, got %q", etag2) t.Errorf("Expected ETag to be cleared on quota limit, got %q", etag2)
+2 -1
View File
@@ -3,12 +3,13 @@ package posthog
import ( import (
"bytes" "bytes"
"context" "context"
"encoding/json"
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
"net/url" "net/url"
"time" "time"
json "github.com/goccy/go-json"
) )
type FlagsRequestData struct { type FlagsRequestData struct {
+1
View File
@@ -13,6 +13,7 @@ require (
require ( require (
github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect
github.com/goccy/go-json v0.10.5 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
+2
View File
@@ -7,6 +7,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6N
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
+1
View File
@@ -74,3 +74,4 @@ func (msg GroupIdentify) APIfy() APIMessage {
return apified return apified
} }
+4 -1
View File
@@ -1,6 +1,8 @@
package posthog package posthog
import "time" import (
"time"
)
var _ Message = (*Identify)(nil) var _ Message = (*Identify)(nil)
@@ -64,3 +66,4 @@ func (msg Identify) APIfy() APIMessage {
return apified return apified
} }
+308
View File
@@ -0,0 +1,308 @@
package posthog
import (
"bytes"
"fmt"
"io"
"net/http"
"net/http/httptest"
"testing"
json "github.com/goccy/go-json"
)
// BenchmarkJSONMarshalBatch benchmarks JSON serialization at various batch sizes
// Uses pre-serialized messages to match production behavior
func BenchmarkJSONMarshalBatch(b *testing.B) {
sizes := []int{1, 10, 50, 100, 250}
for _, size := range sizes {
b.Run(fmt.Sprintf("batch_%d", size), func(b *testing.B) {
// Pre-generate and pre-serialize messages (as in production)
msgs := make([]json.RawMessage, size)
for i := range msgs {
data, _ := json.Marshal(generateVariedCapture(i).APIfy())
msgs[i] = json.RawMessage(data)
}
payload := batch{ApiKey: "test", Messages: msgs}
b.ResetTimer()
for i := 0; i < b.N; i++ {
json.Marshal(payload)
}
})
}
}
// BenchmarkJSONMarshalBatchWithCardinality benchmarks JSON serialization with various property cardinalities
// Uses pre-serialized messages to match production behavior
func BenchmarkJSONMarshalBatchWithCardinality(b *testing.B) {
cardinalities := []struct {
name string
cardinality PropertyCardinality
}{
{"low_10-100_props", CardinalityLow},
{"medium_500-2000_props", CardinalityMedium},
{"high_2000-10000_props", CardinalityHigh},
}
batchSize := 10 // Use smaller batch for high cardinality to avoid excessive memory
for _, tc := range cardinalities {
b.Run(tc.name, func(b *testing.B) {
// Pre-generate and pre-serialize messages with specific cardinality
msgs := make([]json.RawMessage, batchSize)
for i := range msgs {
capture := Capture{
DistinctId: fmt.Sprintf("user_%d", i),
Event: "test_event",
Properties: generatePropertiesWithCardinality(i, tc.cardinality),
Groups: generateGroupsWithCardinality(i, tc.cardinality),
}
data, _ := json.Marshal(capture.APIfy())
msgs[i] = json.RawMessage(data)
}
payload := batch{ApiKey: "test", Messages: msgs}
b.ResetTimer()
for i := 0; i < b.N; i++ {
json.Marshal(payload)
}
})
}
}
// BenchmarkPrepareForSend_Cardinality benchmarks prepareForSend at various property cardinalities
// This validates that the combined APIfy + serialization isn't a bottleneck
func BenchmarkPrepareForSend_Cardinality(b *testing.B) {
cardinalities := []struct {
name string
cardinality PropertyCardinality
}{
{"low_10-100_props", CardinalityLow},
{"medium_500-2000_props", CardinalityMedium},
{"high_2000-10000_props", CardinalityHigh},
}
for _, tc := range cardinalities {
b.Run(tc.name, func(b *testing.B) {
// Pre-generate captures with this cardinality
captures := make([]Capture, 100)
for i := range captures {
captures[i] = Capture{
Type: "capture",
DistinctId: fmt.Sprintf("user_%d", i),
Event: "test_event",
Properties: generatePropertiesWithCardinality(i, tc.cardinality),
Groups: generateGroupsWithCardinality(i, tc.cardinality),
}
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
prepareForSend(captures[i%100])
}
})
}
}
// BenchmarkHTTPUpload benchmarks HTTP upload isolated from serialization
func BenchmarkHTTPUpload(b *testing.B) {
// Pre-serialize payloads
payloadSizes := []int{1024, 10240, 102400} // 1KB, 10KB, 100KB
for _, size := range payloadSizes {
b.Run(fmt.Sprintf("payload_%dKB", size/1024), func(b *testing.B) {
payload := make([]byte, size)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
io.Copy(io.Discard, r.Body)
w.WriteHeader(200)
}))
defer server.Close()
client := &http.Client{}
b.ResetTimer()
for i := 0; i < b.N; i++ {
req, _ := http.NewRequest("POST", server.URL, bytes.NewReader(payload))
resp, _ := client.Do(req)
if resp != nil {
resp.Body.Close()
}
}
})
}
}
// BenchmarkHTTPUploadWithRealPayload benchmarks HTTP upload with realistic JSON payloads
func BenchmarkHTTPUploadWithRealPayload(b *testing.B) {
cardinalities := []struct {
name string
cardinality PropertyCardinality
}{
{"low_cardinality", CardinalityLow},
{"medium_cardinality", CardinalityMedium},
}
for _, tc := range cardinalities {
b.Run(tc.name, func(b *testing.B) {
// Pre-generate and pre-serialize messages (as in production)
msgs := make([]json.RawMessage, 10)
for i := range msgs {
capture := Capture{
DistinctId: fmt.Sprintf("user_%d", i),
Event: "test_event",
Properties: generatePropertiesWithCardinality(i, tc.cardinality),
}
data, _ := json.Marshal(capture.APIfy())
msgs[i] = json.RawMessage(data)
}
payload, _ := json.Marshal(batch{ApiKey: "test", Messages: msgs})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
io.Copy(io.Discard, r.Body)
w.WriteHeader(200)
}))
defer server.Close()
client := &http.Client{}
b.ResetTimer()
b.SetBytes(int64(len(payload)))
for i := 0; i < b.N; i++ {
req, _ := http.NewRequest("POST", server.URL, bytes.NewReader(payload))
req.Header.Set("Content-Type", "application/json")
resp, _ := client.Do(req)
if resp != nil {
resp.Body.Close()
}
}
})
}
}
// BenchmarkPrepareVsMarshaling compares the cost of prepareForSend vs actual marshaling
func BenchmarkPrepareVsMarshaling(b *testing.B) {
cardinalities := []struct {
name string
cardinality PropertyCardinality
}{
{"low", CardinalityLow},
{"medium", CardinalityMedium},
{"high", CardinalityHigh},
}
for _, tc := range cardinalities {
// Pre-generate capture
capture := Capture{
Type: "capture",
DistinctId: "user_1",
Event: "test_event",
Properties: generatePropertiesWithCardinality(42, tc.cardinality),
Groups: generateGroupsWithCardinality(42, tc.cardinality),
}
apiMsg := capture.APIfy()
b.Run(tc.name+"_prepare", func(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
prepareForSend(capture)
}
})
b.Run(tc.name+"_marshal", func(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
json.Marshal(apiMsg)
}
})
}
}
// BenchmarkPropertyIteration benchmarks the cost of iterating over properties
func BenchmarkPropertyIteration(b *testing.B) {
cardinalities := []struct {
name string
cardinality PropertyCardinality
}{
{"low", CardinalityLow},
{"medium", CardinalityMedium},
{"high", CardinalityHigh},
}
for _, tc := range cardinalities {
props := generatePropertiesWithCardinality(42, tc.cardinality)
b.Run(tc.name, func(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
total := 0
for k, v := range props {
total += len(k)
_ = v // access value
}
}
})
}
}
// legacyBatch represents the old batch structure with []APIMessage for comparison benchmarks.
// In the old flow, messages were serialized together with the batch wrapper in a single Marshal call.
type legacyBatch struct {
ApiKey string `json:"api_key"`
HistoricalMigration bool `json:"historical_migration,omitempty"`
Messages []APIMessage `json:"batch"`
}
// BenchmarkOldVsNewSerializationFlow compares serialization approaches:
// - Old: APIfy all messages, then marshal entire batch (single Marshal call)
// - New: prepareForSend each message (marshal individually), then marshal batch wrapper
func BenchmarkOldVsNewSerializationFlow(b *testing.B) {
batchSizes := []int{10, 50, 100, 250}
cardinalities := []struct {
name string
cardinality PropertyCardinality
}{
{"low", CardinalityLow},
{"medium", CardinalityMedium},
}
for _, size := range batchSizes {
for _, tc := range cardinalities {
// Pre-generate captures
captures := make([]Capture, size)
for i := range captures {
captures[i] = Capture{
Type: "capture",
DistinctId: fmt.Sprintf("user_%d", i),
Event: "test_event",
Properties: generatePropertiesWithCardinality(i, tc.cardinality),
Groups: generateGroupsWithCardinality(i, tc.cardinality),
}
}
// Old flow: APIfy then marshal entire batch
b.Run(fmt.Sprintf("old_batch_%d_%s", size, tc.name), func(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
apiMsgs := make([]APIMessage, size)
for j, c := range captures {
apiMsgs[j] = c.APIfy()
}
json.Marshal(legacyBatch{ApiKey: "test", Messages: apiMsgs})
}
})
// New flow: prepareForSend (marshal individually) then marshal batch wrapper
b.Run(fmt.Sprintf("new_batch_%d_%s", size, tc.name), func(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
rawMsgs := make([]json.RawMessage, size)
for j, c := range captures {
data, _, _ := prepareForSend(c)
rawMsgs[j] = data
}
json.Marshal(batch{ApiKey: "test", Messages: rawMsgs})
}
})
}
}
}
-87
View File
@@ -1,87 +0,0 @@
package posthog
import (
"reflect"
"strings"
)
// Imitate what what the JSON package would do when serializing a struct value,
// the only difference is we we don't serialize zero-value struct fields as well.
// Note that this function doesn't recursively convert structures to maps, only
// the value passed as argument is transformed.
func structToMap(v reflect.Value, m map[string]interface{}) map[string]interface{} {
t := v.Type()
n := t.NumField()
if m == nil {
m = make(map[string]interface{}, n)
}
for i := 0; i != n; i++ {
field := t.Field(i)
value := v.Field(i)
name, omitempty := parseJsonTag(field.Tag.Get("json"), field.Name)
if name != "-" && !(omitempty && isZeroValue(value)) {
m[name] = value.Interface()
}
}
return m
}
// Parses a JSON tag the way the json package would do it, returing the expected
// name of the field once serialized and if empty values should be omitted.
func parseJsonTag(tag string, defName string) (name string, omitempty bool) {
args := strings.Split(tag, ",")
if len(args) == 0 || len(args[0]) == 0 {
name = defName
} else {
name = args[0]
}
if len(args) > 1 && args[1] == "omitempty" {
omitempty = true
}
return
}
// Checks if the value given as argument is a zero-value, it is based on the
// isEmptyValue function in https://golang.org/src/encoding/json/encode.go
// but also checks struct types recursively.
func isZeroValue(v reflect.Value) bool {
switch v.Kind() {
case reflect.Array, reflect.Map, reflect.Slice, reflect.String:
return v.Len() == 0
case reflect.Bool:
return !v.Bool()
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return v.Int() == 0
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
return v.Uint() == 0
case reflect.Float32, reflect.Float64:
return v.Float() == 0
case reflect.Interface, reflect.Ptr:
return v.IsNil()
case reflect.Struct:
for i, n := 0, v.NumField(); i != n; i++ {
if !isZeroValue(v.Field(i)) {
return false
}
}
return true
case reflect.Invalid:
return true
}
return false
}
-203
View File
@@ -1,203 +0,0 @@
package posthog
import (
"reflect"
"testing"
)
func TestParseJsonTagEmpty(t *testing.T) {
name, omitempty := parseJsonTag("", "default")
if name != "default" {
t.Error("invalid field name found in empty tag:", name)
}
if omitempty {
t.Error("unexpected 'omitempty' state found in empty tag")
}
}
func TestParseJsonTagName(t *testing.T) {
name, omitempty := parseJsonTag("name", "default")
if name != "name" {
t.Error("invalid field name found in json tag:", name)
}
if omitempty {
t.Error("unexpected 'omitempty' state found in json tag")
}
}
func TestParseJsonTagOmitempty(t *testing.T) {
name, omitempty := parseJsonTag(",omitempty", "default")
if name != "default" {
t.Error("invalid field name found in omitempty tag:", name)
}
if !omitempty {
t.Error("expected 'omitempty' state not found in json tag")
}
}
func TestParseJsonTagNameOmitempty(t *testing.T) {
name, omitempty := parseJsonTag("name,omitempty", "default")
if name != "name" {
t.Error("invalid field name found in json tag:", name)
}
if !omitempty {
t.Error("expected 'omitempty' state not found in json tag")
}
}
func TestStructToMap(t *testing.T) {
type X struct {
A bool
B int `json:"b"`
C string `json:"c,omitempty"`
}
if m := structToMap(reflect.ValueOf(X{}), nil); !reflect.DeepEqual(m, map[string]interface{}{
"A": false,
"b": 0,
}) {
t.Error("invalid JSON object representation of a struct:", m)
}
}
func TestIsZeroValueStringTrue(t *testing.T) {
if !isZeroValue(reflect.ValueOf("")) {
t.Error("empty string should be a zero-value")
}
}
func TestIsZeroValueStringFalse(t *testing.T) {
if isZeroValue(reflect.ValueOf("A")) {
t.Error("non-empty string should not be a zero-value")
}
}
func TestIsZeroValueSliceTrue(t *testing.T) {
if !isZeroValue(reflect.ValueOf([]int{})) {
t.Error("empty slice should be a zero-value")
}
}
func TestIsZeroValueSliceFalse(t *testing.T) {
if isZeroValue(reflect.ValueOf([]int{0})) {
t.Error("non-empty slice should not be a zero-value")
}
}
func TestIsZeroValueMapTrue(t *testing.T) {
if !isZeroValue(reflect.ValueOf(map[string]string{})) {
t.Error("empty map should be a zero-value")
}
}
func TestIsZeroValueMapFalse(t *testing.T) {
if isZeroValue(reflect.ValueOf(map[string]string{
"A": "a",
})) {
t.Error("non-empty map should not be a zero-value")
}
}
func TestIsZeroValueBoolTrue(t *testing.T) {
if !isZeroValue(reflect.ValueOf(false)) {
t.Error("`false` should be a zero-value")
}
}
func TestIsZeroValueBoolFalse(t *testing.T) {
if isZeroValue(reflect.ValueOf(true)) {
t.Error("`true` should not be a zero-value")
}
}
func TestIsZeroValueIntTrue(t *testing.T) {
if !isZeroValue(reflect.ValueOf(0)) {
t.Error("`0` should be a zero-value")
}
}
func TestIsZeroValueIntFalse(t *testing.T) {
if isZeroValue(reflect.ValueOf(1)) {
t.Error("`1` should not be a zero-value")
}
}
func TestIsZeroValueUintTrue(t *testing.T) {
if !isZeroValue(reflect.ValueOf(uint(0))) {
t.Error("`0` should be a zero-value")
}
}
func TestIsZeroValueUintFalse(t *testing.T) {
if isZeroValue(reflect.ValueOf(uint(1))) {
t.Error("`1` should not be a zero-value")
}
}
func TestIsZeroValueFloatTrue(t *testing.T) {
if !isZeroValue(reflect.ValueOf(0.0)) {
t.Error("`0.0` should be a zero-value")
}
}
func TestIsZeroValueFloatFalse(t *testing.T) {
if isZeroValue(reflect.ValueOf(1.0)) {
t.Error("`1.0` should not be a zero-value")
}
}
func TestIsZeroValuePointerTrue(t *testing.T) {
if !isZeroValue(reflect.ValueOf((*int)(nil))) {
t.Error("nil pointer should be a zero-value")
}
}
func TestIsZeroValuePointerFalse(t *testing.T) {
if isZeroValue(reflect.ValueOf(t)) {
t.Error("non-nil pointer should not be a zero-value")
}
}
func TestIsZeroValueStructTrue(t *testing.T) {
type T struct {
A bool
B int
C string
}
if !isZeroValue(reflect.ValueOf(T{})) {
t.Error("empty struct should be a zero-value")
}
}
func TestIsZeroValueStructFalse(t *testing.T) {
type T struct {
A bool
B int
C string
}
if isZeroValue(reflect.ValueOf(T{A: true})) {
t.Error("non-empty struct should not be a zero-value")
}
}
func TestIsZeroValueNil(t *testing.T) {
if !isZeroValue(reflect.ValueOf(nil)) {
t.Error("nil should be a zero-value")
}
}
func TestIsZeroValueFunc(t *testing.T) {
if isZeroValue(reflect.ValueOf(func() {})) {
t.Error("functions should be be zero-value")
}
}
+16 -58
View File
@@ -1,8 +1,9 @@
package posthog package posthog
import ( import (
"encoding/json"
"time" "time"
json "github.com/goccy/go-json"
) )
// Values implementing this interface are used by posthog clients to notify // Values implementing this interface are used by posthog clients to notify
@@ -51,71 +52,28 @@ func makeTimestamp(t time.Time, def time.Time) time.Time {
return t return t
} }
// This structure represents objects sent to the /batch/ endpoint. We don't // batch represents objects sent to the /batch/ endpoint with pre-serialized messages.
// export this type because it's only meant to be used internally to send groups // Messages are pre-serialized as json.RawMessage for efficient batch building -
// of messages in one API call. // json.Marshal embeds them directly without re-encoding.
type batch struct { type batch struct {
ApiKey string `json:"api_key"` ApiKey string `json:"api_key"`
HistoricalMigration bool `json:"historical_migration,omitempty"` HistoricalMigration bool `json:"historical_migration,omitempty"`
Messages []message `json:"batch"` Messages []json.RawMessage `json:"batch"`
} }
type APIMessage interface{} type APIMessage interface{}
type message struct { // prepareForSend creates the API message and serializes it to JSON.
msg APIMessage // Returns pre-serialized JSON for efficient batch building, the original
json []byte // APIMessage for callbacks, and any serialization error.
} // Size is derived from len(json.RawMessage) when needed - O(1) operation.
func prepareForSend(msg Message) (json.RawMessage, APIMessage, error) {
func makeMessage(m APIMessage, maxBytes int) (msg message, err error) { apiMsg := msg.APIfy()
if msg.json, err = json.Marshal(m); err == nil { data, err := json.Marshal(apiMsg)
if len(msg.json) > maxBytes { if err != nil {
err = ErrMessageTooBig return nil, apiMsg, err
} else {
msg.msg = m
} }
} return json.RawMessage(data), apiMsg, nil
return
}
func (m message) MarshalJSON() ([]byte, error) {
return m.json, nil
}
func (m message) size() int {
// The `+ 1` is for the comma that sits between each items of a JSON array.
return len(m.json) + 1
}
type messageQueue struct {
pending []message
bytes int
maxBatchSize int
maxBatchBytes int
}
func (q *messageQueue) push(m message) (b []message) {
if (q.bytes + m.size()) > q.maxBatchBytes {
b = q.flush()
}
if q.pending == nil {
q.pending = make([]message, 0, q.maxBatchSize)
}
q.pending = append(q.pending, m)
q.bytes += len(m.json)
if b == nil && len(q.pending) == q.maxBatchSize {
b = q.flush()
}
return
}
func (q *messageQueue) flush() (msgs []message) {
msgs, q.pending, q.bytes = q.pending, nil, 0
return
} }
const ( const (
-66
View File
@@ -1,66 +0,0 @@
package posthog
import (
"reflect"
"testing"
)
func TestMessageQueuePushMaxBatchSize(t *testing.T) {
m0, _ := makeMessage(Capture{
DistinctId: "1",
Event: "A",
}, maxMessageBytes)
m1, _ := makeMessage(Capture{
DistinctId: "2",
Event: "A",
}, maxMessageBytes)
q := messageQueue{
maxBatchSize: 2,
maxBatchBytes: maxBatchBytes,
}
if msgs := q.push(m0); msgs != nil {
t.Error("unexpected message batch returned after pushing only one message")
}
if msgs := q.push(m1); !reflect.DeepEqual(msgs, []message{m0, m1}) {
t.Error("invalid message batch returned after pushing two messages:", msgs)
}
}
func TestMessageQueuePushMaxBatchBytes(t *testing.T) {
m0, _ := makeMessage(Capture{
DistinctId: "1",
Event: "A",
}, maxMessageBytes)
m1, _ := makeMessage(Capture{
DistinctId: "2",
Event: "A",
}, maxMessageBytes)
q := messageQueue{
maxBatchSize: 100,
maxBatchBytes: len(m0.json) + 1,
}
if msgs := q.push(m0); msgs != nil {
t.Error("unexpected message batch returned after pushing only one message")
}
if msgs := q.push(m1); !reflect.DeepEqual(msgs, []message{m0}) {
t.Error("invalid message batch returned after pushing two messages:", msgs)
}
if !reflect.DeepEqual(q.pending, []message{m1}) {
t.Error("invalid state of the message queue after pushing two messages:", q.pending)
}
}
func TestMakeMessageTooBig(t *testing.T) {
if _, err := makeMessage(Capture{DistinctId: "1"}, 1); err != ErrMessageTooBig {
t.Error("invalid error returned when creating a message bigger than the limit:", err)
}
}
+551
View File
@@ -0,0 +1,551 @@
package posthog
import (
"io"
json "github.com/goccy/go-json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/stretchr/testify/require"
)
// TestEnqueue_InvalidMessageTypes tests various invalid message scenarios
func TestEnqueue_InvalidMessageTypes(t *testing.T) {
client, _ := NewWithConfig("test-key", Config{
Transport: NoOpTransport(),
})
defer client.Close()
t.Run("capture_missing_distinct_id", func(t *testing.T) {
err := client.Enqueue(Capture{
Event: "test_event",
})
require.Error(t, err)
require.Contains(t, err.Error(), "DistinctId")
})
t.Run("capture_missing_event", func(t *testing.T) {
err := client.Enqueue(Capture{
DistinctId: "user_1",
})
require.Error(t, err)
require.Contains(t, err.Error(), "Event")
})
t.Run("identify_missing_distinct_id", func(t *testing.T) {
err := client.Enqueue(Identify{
Properties: Properties{"name": "test"},
})
require.Error(t, err)
})
t.Run("alias_missing_distinct_id", func(t *testing.T) {
err := client.Enqueue(Alias{
Alias: "new_alias",
})
require.Error(t, err)
})
t.Run("alias_missing_alias", func(t *testing.T) {
err := client.Enqueue(Alias{
DistinctId: "user_1",
})
require.Error(t, err)
})
t.Run("group_identify_missing_type", func(t *testing.T) {
err := client.Enqueue(GroupIdentify{
Key: "group_1",
})
require.Error(t, err)
})
t.Run("group_identify_missing_key", func(t *testing.T) {
err := client.Enqueue(GroupIdentify{
Type: "company",
})
require.Error(t, err)
})
}
// TestFeatureFlag_MalformedResponses tests handling of malformed feature flag responses
func TestFeatureFlag_MalformedResponses(t *testing.T) {
malformedResponses := []struct {
name string
response string
}{
{"invalid_json", `{invalid json`},
{"flags_not_object", `{"featureFlags": "not an object"}`},
{"empty_response", ``},
{"null_response", `null`},
{"array_instead_of_object", `[]`},
}
for _, tc := range malformedResponses {
t.Run(tc.name, func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
w.Write([]byte(tc.response))
}))
defer server.Close()
client, _ := NewWithConfig("test-key", Config{
Endpoint: server.URL,
PersonalApiKey: "test-personal",
})
defer client.Close()
// Should not panic, should handle gracefully
result, err := client.GetFeatureFlag(FeatureFlagPayload{
Key: "test-flag",
DistinctId: "user_1",
})
// Either returns error or returns false/nil - shouldn't crash
t.Logf("Result: %v, Error: %v", result, err)
})
}
}
// TestClient_ServerErrors tests client behavior with various server error codes
func TestClient_ServerErrors(t *testing.T) {
errorCodes := []int{
http.StatusBadRequest,
http.StatusUnauthorized,
http.StatusForbidden,
http.StatusNotFound,
http.StatusInternalServerError,
http.StatusBadGateway,
http.StatusServiceUnavailable,
http.StatusGatewayTimeout,
}
for _, code := range errorCodes {
t.Run(http.StatusText(code), func(t *testing.T) {
callCount := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
callCount++
w.WriteHeader(code)
}))
defer server.Close()
maxRetries := 0
client, _ := NewWithConfig("test-key", Config{
Endpoint: server.URL,
MaxRetries: &maxRetries, // Disable retries for faster test
})
err := client.Enqueue(Capture{
DistinctId: "user_1",
Event: "test_event",
})
require.NoError(t, err) // Enqueue should not fail immediately
client.Close() // Will attempt to flush
t.Logf("Server received %d request(s) for status %d", callCount, code)
})
}
}
// TestClient_NetworkFailures tests client behavior with network failures
func TestClient_NetworkFailures(t *testing.T) {
t.Run("connection_refused", func(t *testing.T) {
client, _ := NewWithConfig("test-key", Config{
Endpoint: "http://localhost:1", // Port 1 should refuse connections
MaxRetries: Ptr(0), // Skip retries to avoid timeout
})
err := client.Enqueue(Capture{
DistinctId: "user_1",
Event: "test_event",
})
require.NoError(t, err) // Enqueue should not fail immediately
// Close will try to flush and should handle the error gracefully
err = client.Close()
require.NoError(t, err)
})
t.Run("invalid_url", func(t *testing.T) {
client, _ := NewWithConfig("test-key", Config{
Endpoint: "not-a-valid-url",
MaxRetries: Ptr(0), // Skip retries to avoid timeout
})
err := client.Enqueue(Capture{
DistinctId: "user_1",
Event: "test_event",
})
require.NoError(t, err)
err = client.Close()
require.NoError(t, err)
})
}
// TestProperties_EdgeCases tests edge cases in property handling
func TestProperties_EdgeCases(t *testing.T) {
t.Run("nil_value", func(t *testing.T) {
props := Properties{"key": nil}
// Should serialize without error
_, err := json.Marshal(props)
require.NoError(t, err)
})
t.Run("empty_key", func(t *testing.T) {
props := Properties{"": "empty key"}
_, err := json.Marshal(props)
require.NoError(t, err)
})
t.Run("very_large_string", func(t *testing.T) {
largeString := strings.Repeat("x", 1000000)
props := Properties{"key": largeString}
data, err := json.Marshal(props)
require.NoError(t, err)
require.True(t, len(data) > 1000000)
})
t.Run("special_characters", func(t *testing.T) {
props := Properties{
"unicode": "日本語テスト",
"emoji": "🚀🔥💡",
"newlines": "line1\nline2\nline3",
"tabs": "col1\tcol2\tcol3",
"quotes": `"quoted"`,
"backslash": `path\to\file`,
"null_bytes": "before\x00after",
}
data, err := json.Marshal(props)
require.NoError(t, err)
// Should be valid JSON that can be unmarshaled
var decoded map[string]interface{}
err = json.Unmarshal(data, &decoded)
require.NoError(t, err)
})
t.Run("deeply_nested", func(t *testing.T) {
nested := map[string]interface{}{}
current := nested
for i := 0; i < 100; i++ {
next := map[string]interface{}{}
current["nested"] = next
current = next
}
current["value"] = "deep"
props := Properties{"deep": nested}
_, err := json.Marshal(props)
require.NoError(t, err)
})
t.Run("circular_reference_avoided", func(t *testing.T) {
// Properties using map[string]interface{} can't have circular refs
// but we can test that large recursive structures work
props := Properties{
"a": map[string]interface{}{
"b": map[string]interface{}{
"c": "value",
},
},
}
_, err := json.Marshal(props)
require.NoError(t, err)
})
}
// TestConfig_Validation tests configuration validation
func TestConfig_Validation(t *testing.T) {
t.Run("negative_batch_size", func(t *testing.T) {
// SDK validates and rejects negative batch sizes
_, err := NewWithConfig("test-key", Config{
BatchSize: -1,
Transport: NoOpTransport(),
})
require.Error(t, err, "negative batch size should be rejected")
})
t.Run("zero_batch_size", func(t *testing.T) {
// Zero batch size is valid - SDK uses default
client, err := NewWithConfig("test-key", Config{
BatchSize: 0,
Transport: NoOpTransport(),
})
require.NoError(t, err)
defer client.Close()
})
t.Run("empty_api_key", func(t *testing.T) {
// Note: SDK currently accepts empty API keys (server will reject them)
// This test documents current behavior
client, err := NewWithConfig("", Config{
Transport: NoOpTransport(),
})
require.NoError(t, err, "SDK accepts empty API key (server-side validation)")
defer client.Close()
})
}
// TestFeatureFlag_TimeoutHandling tests flag evaluation with network delays
func TestFeatureFlag_TimeoutHandling(t *testing.T) {
t.Run("slow_response", func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(100 * time.Millisecond)
w.Write([]byte(`{"featureFlags": {"test-flag": true}}`))
}))
defer server.Close()
client, _ := NewWithConfig("test-key", Config{
Endpoint: server.URL,
PersonalApiKey: "test-personal",
})
defer client.Close()
start := time.Now()
_, err := client.GetFeatureFlag(FeatureFlagPayload{
Key: "test-flag",
DistinctId: "user_1",
})
elapsed := time.Since(start)
t.Logf("Request took %v, error: %v", elapsed, err)
})
}
// TestEnqueue_AfterClose tests behavior when enqueueing after client is closed
func TestEnqueue_AfterClose(t *testing.T) {
client, _ := NewWithConfig("test-key", Config{
Transport: NoOpTransport(),
})
// Close the client
client.Close()
// Try to enqueue - should not panic, should return error or handle gracefully
err := client.Enqueue(Capture{
DistinctId: "user_1",
Event: "test_event",
})
// The SDK might allow this or return an error - either is acceptable
t.Logf("Enqueue after close result: %v", err)
}
// TestCallback_Errors tests that callback errors are handled gracefully
func TestCallback_Errors(t *testing.T) {
callbackCalled := false
panicCallback := struct {
Callback
}{}
// Create a callback that panics
type panicyCallback struct{}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(200)
}))
defer server.Close()
callback := NewUnifiedCallback(t)
client, _ := NewWithConfig("test-key", Config{
Endpoint: server.URL,
Callback: callback,
})
err := client.Enqueue(Capture{
DistinctId: "user_1",
Event: "test_event",
})
require.NoError(t, err)
client.Close()
callbackCalled = callback.successCount > 0 || callback.failureCount > 0
t.Logf("Callback was called: %v, panic callback: %v", callbackCalled, panicCallback)
}
// TestBatch_EmptyAndLarge tests batching edge cases
func TestBatch_EmptyAndLarge(t *testing.T) {
t.Run("large_batch", func(t *testing.T) {
var received int
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, _ := io.ReadAll(r.Body)
var b struct {
Messages []json.RawMessage `json:"batch"`
}
json.Unmarshal(body, &b)
received += len(b.Messages)
w.WriteHeader(200)
}))
defer server.Close()
client, _ := NewWithConfig("test-key", Config{
Endpoint: server.URL,
BatchSize: 1000,
})
// Send more than default batch size
for i := 0; i < 500; i++ {
client.Enqueue(Capture{
DistinctId: "user_1",
Event: "test_event",
Properties: Properties{"index": i},
})
}
client.Close()
require.Equal(t, 500, received)
})
}
// TestPrepareForSend_EdgeCases tests prepareForSend with edge case data
func TestPrepareForSend_EdgeCases(t *testing.T) {
t.Run("empty_capture", func(t *testing.T) {
capture := Capture{Type: "capture"}
data, apiMsg, err := prepareForSend(capture)
require.NoError(t, err)
require.Greater(t, len(data), 0, "Even empty capture should have non-zero serialized size")
require.NotNil(t, apiMsg, "APIMessage should not be nil")
})
t.Run("nil_properties_and_groups", func(t *testing.T) {
capture := Capture{
Type: "capture",
DistinctId: "user",
Event: "event",
Properties: nil,
Groups: nil,
}
data, apiMsg, err := prepareForSend(capture)
require.NoError(t, err)
require.Greater(t, len(data), 0)
require.NotNil(t, apiMsg)
})
t.Run("empty_properties_and_groups", func(t *testing.T) {
capture := Capture{
Type: "capture",
DistinctId: "user",
Event: "event",
Properties: Properties{},
Groups: Groups{},
}
data, apiMsg, err := prepareForSend(capture)
require.NoError(t, err)
require.Greater(t, len(data), 0)
require.NotNil(t, apiMsg)
})
}
// TestPrepareForSend_SerializationErrors tests that prepareForSend returns errors for unencodable values
func TestPrepareForSend_SerializationErrors(t *testing.T) {
t.Run("channel_in_properties", func(t *testing.T) {
ch := make(chan int)
capture := Capture{
Type: "capture",
DistinctId: "user",
Event: "event",
Properties: Properties{"channel": ch},
}
data, apiMsg, err := prepareForSend(capture)
require.Error(t, err, "Should fail to serialize channel")
require.Nil(t, data, "Data should be nil on error")
require.NotNil(t, apiMsg, "APIMessage should be returned for callback")
})
t.Run("function_in_properties", func(t *testing.T) {
fn := func() {}
capture := Capture{
Type: "capture",
DistinctId: "user",
Event: "event",
Properties: Properties{"func": fn},
}
data, apiMsg, err := prepareForSend(capture)
require.Error(t, err, "Should fail to serialize function")
require.Nil(t, data)
require.NotNil(t, apiMsg, "APIMessage should be returned for callback")
})
t.Run("channel_in_groups", func(t *testing.T) {
ch := make(chan string)
capture := Capture{
Type: "capture",
DistinctId: "user",
Event: "event",
Groups: Groups{"company": ch},
}
data, apiMsg, err := prepareForSend(capture)
require.Error(t, err, "Should fail to serialize channel in groups")
require.Nil(t, data)
require.NotNil(t, apiMsg, "APIMessage should be returned for callback")
})
t.Run("identify_with_unencodable", func(t *testing.T) {
ch := make(chan int)
identify := Identify{
DistinctId: "user",
Properties: Properties{"channel": ch},
}
data, apiMsg, err := prepareForSend(identify)
require.Error(t, err, "Identify should fail with unencodable value")
require.Nil(t, data)
require.NotNil(t, apiMsg, "APIMessage should be returned for callback")
})
t.Run("alias_serializes_ok", func(t *testing.T) {
// Alias has no Properties, so should always serialize successfully
alias := Alias{
DistinctId: "user",
Alias: "alias",
}
data, apiMsg, err := prepareForSend(alias)
require.NoError(t, err, "Alias should serialize successfully")
require.NotNil(t, data)
require.NotNil(t, apiMsg)
})
t.Run("group_identify_with_unencodable", func(t *testing.T) {
fn := func() {}
groupIdentify := GroupIdentify{
Type: "company",
Key: "company_1",
Properties: Properties{"func": fn},
}
data, apiMsg, err := prepareForSend(groupIdentify)
require.Error(t, err, "GroupIdentify should fail with unencodable value")
require.Nil(t, data)
require.NotNil(t, apiMsg, "APIMessage should be returned for callback")
})
}
// TestGroups_EdgeCases tests group property edge cases
func TestGroups_EdgeCases(t *testing.T) {
t.Run("empty_group_type", func(t *testing.T) {
groups := Groups{"": "value"}
_, err := json.Marshal(groups)
require.NoError(t, err)
})
t.Run("empty_group_value", func(t *testing.T) {
groups := Groups{"type": ""}
_, err := json.Marshal(groups)
require.NoError(t, err)
})
t.Run("many_groups", func(t *testing.T) {
groups := Groups{}
for i := 0; i < 100; i++ {
groups[string(rune('a'+i%26))+string(rune('0'+i/26))] = "value"
}
_, err := json.Marshal(groups)
require.NoError(t, err)
})
}
+398 -151
View File
@@ -2,17 +2,18 @@ package posthog
import ( import (
"bytes" "bytes"
"encoding/json" "context"
"errors" "errors"
"fmt" "fmt"
"io" "io"
"io/ioutil"
"net/http" "net/http"
"net/url" "net/url"
"sync" "sync"
"sync/atomic"
"time" "time"
lru "github.com/hashicorp/golang-lru/v2" json "github.com/goccy/go-json"
"github.com/hashicorp/golang-lru/v2"
) )
const ( const (
@@ -20,6 +21,13 @@ const (
CACHE_DEFAULT_SIZE = 300_000 CACHE_DEFAULT_SIZE = 300_000
propertyGeoipDisable = "$geoip_disable" propertyGeoipDisable = "$geoip_disable"
// DefaultIdleConns is the default max idle connections for the HTTP client.
DefaultIdleConns = 100
// DefaultIdleConnsPerHost is the default max idle connections per host.
// This is higher than http.DefaultTransport's value of 2 to reduce
// connection churn when sending batches.
DefaultIdleConnsPerHost = 10
) )
type EnqueueClient interface { type EnqueueClient interface {
@@ -71,8 +79,25 @@ type Client interface {
// NB: This is only available when using a PersonalApiKey // NB: This is only available when using a PersonalApiKey
GetFeatureFlags() ([]FeatureFlag, error) GetFeatureFlags() ([]FeatureFlag, error)
// GetLastCapturedEvent returns the last captured event // CloseWithContext gracefully shuts down the client with the provided context.
GetLastCapturedEvent() *Capture // The context can be used to control the shutdown deadline.
CloseWithContext(context.Context) error
}
// preparedMessage bundles a pre-serialized message with the original APIMessage.
// The data field contains pre-serialized JSON for efficient batch building.
// The msg field retains the original APIMessage for callbacks.
// Size is obtained via len(data) - O(1) since json.RawMessage is []byte.
type preparedMessage struct {
data json.RawMessage // pre-serialized JSON for batch submission
msg APIMessage // original message for callbacks
}
// preparedBatch holds both raw data for efficient serialization and
// original API messages for callbacks.
type preparedBatch struct {
data []json.RawMessage // pre-serialized messages for batch submission
msgs []APIMessage // original messages for callbacks
} }
type client struct { type client struct {
@@ -81,8 +106,16 @@ type client struct {
// This channel is where the `Enqueue` method writes messages so they can be // This channel is where the `Enqueue` method writes messages so they can be
// picked up and pushed by the backend goroutine taking care of applying the // picked up and pushed by the backend goroutine taking care of applying the
// batching rules. // batching rules. Messages are pre-converted to APIMessage format with
msgs chan APIMessage // pre-computed size to avoid race conditions.
msgs chan preparedMessage
// Channel for sending batches to workers. Acts as both a queue and
// concurrency limiter - when full, new batches are shed via failure callback.
batches chan preparedBatch
// Tracks in-flight batches for graceful shutdown
inFlight atomic.Int64
// These two channels are used to synchronize the client shutting down when // These two channels are used to synchronize the client shutting down when
// `Close` is called. // `Close` is called.
@@ -92,6 +125,18 @@ type client struct {
quit chan struct{} quit chan struct{}
shutdown chan struct{} shutdown chan struct{}
// Context and cancel function for graceful shutdown.
// When Close is called, the context is cancelled to signal all goroutines
// to stop, including in-flight HTTP requests.
ctx context.Context
cancel context.CancelFunc
// closeOnce ensures Close is idempotent
closeOnce sync.Once
// closed is set to true when the client is closed, used to fast-fail Enqueue
closed atomic.Bool
// This HTTP client is used to send requests to the backend, it uses the // This HTTP client is used to send requests to the backend, it uses the
// HTTP transport provided in the configuration. // HTTP transport provided in the configuration.
http http.Client http http.Client
@@ -101,11 +146,6 @@ type client struct {
distinctIdsFeatureFlagsReported *lru.Cache[flagUser, struct{}] distinctIdsFeatureFlagsReported *lru.Cache[flagUser, struct{}]
// Last captured event
lastCapturedEvent *Capture
// Mutex to protect last captured event
lastEventMutex sync.RWMutex
// Decider for feature flag methods // Decider for feature flag methods
decider decider decider decider
} }
@@ -139,13 +179,24 @@ func NewWithConfig(apiKey string, config Config) (cli Client, err error) {
if err != nil && config.Logger != nil { if err != nil && config.Logger != nil {
config.Logger.Errorf("Error creating cache for reported flags: %v", err) config.Logger.Errorf("Error creating cache for reported flags: %v", err)
} }
// Channel sizing:
// - batches queue sized by MaxEnqueuedRequests (default 1000)
// - msgs queue sized to hold multiple batches worth of messages
batchesQueueSize := config.MaxEnqueuedRequests
msgQueueSize := max(100, config.BatchSize*10)
ctx, cancel := context.WithCancel(context.Background())
c := &client{ c := &client{
Config: config, Config: config,
key: apiKey, key: apiKey,
msgs: make(chan APIMessage, 100), msgs: make(chan preparedMessage, msgQueueSize),
batches: make(chan preparedBatch, batchesQueueSize),
quit: make(chan struct{}), quit: make(chan struct{}),
shutdown: make(chan struct{}), shutdown: make(chan struct{}),
http: makeHttpClient(config.Transport), ctx: ctx,
cancel: cancel,
http: makeHttpClient(config.Transport, config.BatchUploadTimeout),
distinctIdsFeatureFlagsReported: reportedCache, distinctIdsFeatureFlagsReported: reportedCache,
} }
@@ -178,12 +229,20 @@ func NewWithConfig(apiKey string, config Config) (cli Client, err error) {
return return
} }
func makeHttpClient(transport http.RoundTripper) http.Client { func makeHttpClient(transport http.RoundTripper, timeout time.Duration) http.Client {
// If no custom transport provided, clone DefaultTransport with tuned connection pool
if transport == nil {
t := http.DefaultTransport.(*http.Transport).Clone()
t.MaxIdleConns = DefaultIdleConns
t.MaxIdleConnsPerHost = DefaultIdleConnsPerHost
transport = t
}
httpClient := http.Client{ httpClient := http.Client{
Transport: transport, Transport: transport,
} }
if supportsTimeout(transport) { if supportsTimeout(transport) {
httpClient.Timeout = 10 * time.Second httpClient.Timeout = timeout
} }
return httpClient return httpClient
} }
@@ -221,6 +280,11 @@ func dereferenceMessage(msg Message) Message {
} }
func (c *client) Enqueue(msg Message) (err error) { func (c *client) Enqueue(msg Message) (err error) {
// Fast path: check if client is closed before doing any work
if c.closed.Load() {
return ErrClosed
}
msg = dereferenceMessage(msg) msg = dereferenceMessage(msg)
if err = msg.Validate(); err != nil { if err = msg.Validate(); err != nil {
return return
@@ -228,23 +292,55 @@ func (c *client) Enqueue(msg Message) (err error) {
var ts = c.now() var ts = c.now()
// Helper to send prepared message with panic recovery
sendPrepared := func(prepared preparedMessage) {
defer func() {
// When the `msgs` channel is closed writing to it will trigger a panic.
// To avoid letting the panic propagate to the caller we recover from it
// and instead report that the client has been closed and shouldn't be
// used anymore.
if recover() != nil {
err = ErrClosed
}
}()
c.msgs <- prepared
}
switch m := msg.(type) { switch m := msg.(type) {
case Alias: case Alias:
m.Type = "alias" m.Type = "alias"
m.Timestamp = makeTimestamp(m.Timestamp, ts) m.Timestamp = makeTimestamp(m.Timestamp, ts)
m.DisableGeoIP = c.GetDisableGeoIP() m.DisableGeoIP = c.GetDisableGeoIP()
msg = m data, apiMsg, serErr := prepareForSend(m)
if serErr != nil {
c.notifyFailure([]APIMessage{apiMsg}, serErr)
return
}
sendPrepared(preparedMessage{data: data, msg: apiMsg})
return
case Identify: case Identify:
m.Type = "identify" m.Type = "identify"
m.Timestamp = makeTimestamp(m.Timestamp, ts) m.Timestamp = makeTimestamp(m.Timestamp, ts)
m.DisableGeoIP = c.GetDisableGeoIP() m.DisableGeoIP = c.GetDisableGeoIP()
msg = m data, apiMsg, serErr := prepareForSend(m)
if serErr != nil {
c.notifyFailure([]APIMessage{apiMsg}, serErr)
return
}
sendPrepared(preparedMessage{data: data, msg: apiMsg})
return
case GroupIdentify: case GroupIdentify:
m.Timestamp = makeTimestamp(m.Timestamp, ts) m.Timestamp = makeTimestamp(m.Timestamp, ts)
m.DisableGeoIP = c.GetDisableGeoIP() m.DisableGeoIP = c.GetDisableGeoIP()
msg = m data, apiMsg, serErr := prepareForSend(m)
if serErr != nil {
c.notifyFailure([]APIMessage{apiMsg}, serErr)
return
}
sendPrepared(preparedMessage{data: data, msg: apiMsg})
return
case Capture: case Capture:
m.Type = "capture" m.Type = "capture"
@@ -291,50 +387,30 @@ func (c *client) Enqueue(msg Message) (err error) {
m.Properties = NewProperties() m.Properties = NewProperties()
} }
m.Properties.Merge(c.DefaultEventProperties) m.Properties.Merge(c.DefaultEventProperties)
c.setLastCapturedEvent(m) data, apiMsg, serErr := prepareForSend(m)
msg = m if serErr != nil {
c.notifyFailure([]APIMessage{apiMsg}, serErr)
return
}
sendPrepared(preparedMessage{data: data, msg: apiMsg})
return
case Exception: case Exception:
m.Type = "exception" m.Type = "exception"
m.Timestamp = makeTimestamp(m.Timestamp, ts) m.Timestamp = makeTimestamp(m.Timestamp, ts)
m.DisableGeoIP = c.GetDisableGeoIP() m.DisableGeoIP = c.GetDisableGeoIP()
msg = m data, apiMsg, serErr := prepareForSend(m)
if serErr != nil {
c.notifyFailure([]APIMessage{apiMsg}, serErr)
return
}
sendPrepared(preparedMessage{data: data, msg: apiMsg})
return
default: default:
err = fmt.Errorf("messages with custom types cannot be enqueued: %T", msg) err = fmt.Errorf("messages with custom types cannot be enqueued: %T", msg)
return return
} }
defer func() {
// When the `msgs` channel is closed writing to it will trigger a panic.
// To avoid letting the panic propagate to the caller we recover from it
// and instead report that the client has been closed and shouldn't be
// used anymore.
if recover() != nil {
err = ErrClosed
}
}()
c.msgs <- msg.APIfy()
return
}
func (c *client) setLastCapturedEvent(event Capture) {
c.lastEventMutex.Lock()
defer c.lastEventMutex.Unlock()
c.lastCapturedEvent = &event
}
func (c *client) GetLastCapturedEvent() *Capture {
c.lastEventMutex.RLock()
defer c.lastEventMutex.RUnlock()
if c.lastCapturedEvent == nil {
return nil
}
// Return a copy to avoid data races
eventCopy := *c.lastCapturedEvent
return &eventCopy
} }
func (c *client) IsFeatureEnabled(flagConfig FeatureFlagPayload) (interface{}, error) { func (c *client) IsFeatureEnabled(flagConfig FeatureFlagPayload) (interface{}, error) {
@@ -382,6 +458,18 @@ func (c *client) GetFeatureFlagPayload(flagConfig FeatureFlagPayload) (string, e
} }
func (c *client) GetFeatureFlag(flagConfig FeatureFlagPayload) (interface{}, error) { func (c *client) GetFeatureFlag(flagConfig FeatureFlagPayload) (interface{}, error) {
return c.getFeatureFlagWithContext(context.Background(), flagConfig)
}
// getFeatureFlagWithContext returns variant value if multivariant flag or otherwise a boolean
// indicating if the given flag is on or off for the user.
// The context can be used to control timeouts and cancellation.
func (c *client) getFeatureFlagWithContext(ctx context.Context, flagConfig FeatureFlagPayload) (interface{}, error) {
// Check context before starting
if err := ctx.Err(); err != nil {
return false, err
}
if err := flagConfig.validate(); err != nil { if err := flagConfig.validate(); err != nil {
return false, err return false, err
} }
@@ -411,6 +499,11 @@ func (c *client) GetFeatureFlag(flagConfig FeatureFlagPayload) (interface{}, err
} }
} }
// Check context after flag evaluation
if ctx.Err() != nil {
return false, ctx.Err()
}
cacheKey := flagUser{flagConfig.DistinctId, flagConfig.Key} cacheKey := flagUser{flagConfig.DistinctId, flagConfig.Key}
if *flagConfig.SendFeatureFlagEvents && !c.distinctIdsFeatureFlagsReported.Contains(cacheKey) { if *flagConfig.SendFeatureFlagEvents && !c.distinctIdsFeatureFlagsReported.Contains(cacheKey) {
var properties = NewProperties(). var properties = NewProperties().
@@ -456,9 +549,6 @@ func (c *client) GetRemoteConfigPayload(flagKey string) (string, error) {
return c.makeRemoteConfigRequest(flagKey) return c.makeRemoteConfigRequest(flagKey)
} }
// ErrNoPersonalAPIKey is returned when oen tries to use feature flags without specifying a PersonalAPIKey
var ErrNoPersonalAPIKey = errors.New("no PersonalAPIKey provided")
// GetFeatureFlags returns all feature flag definitions used for local evaluation // GetFeatureFlags returns all feature flag definitions used for local evaluation
// This is only available when using a PersonalApiKey. Not to be confused with // This is only available when using a PersonalApiKey. Not to be confused with
// GetAllFlags, which returns all flags and their values for a given user. // GetAllFlags, which returns all flags and their values for a given user.
@@ -476,6 +566,18 @@ func (c *client) GetFeatureFlags() ([]FeatureFlag, error) {
// This first attempts local evaluation if a poller exists, otherwise it falls // This first attempts local evaluation if a poller exists, otherwise it falls
// back to the flags endpoint // back to the flags endpoint
func (c *client) GetAllFlags(flagConfig FeatureFlagPayloadNoKey) (map[string]interface{}, error) { func (c *client) GetAllFlags(flagConfig FeatureFlagPayloadNoKey) (map[string]interface{}, error) {
return c.getAllFlagsWithContext(context.Background(), flagConfig)
}
// getAllFlagsWithContext returns all flags and their values for a given user.
// The context can be used to control timeouts and cancellation.
// A flag value is either a boolean or a variant string (for multivariate flags)
func (c *client) getAllFlagsWithContext(ctx context.Context, flagConfig FeatureFlagPayloadNoKey) (map[string]interface{}, error) {
// Check context before starting
if err := ctx.Err(); err != nil {
return nil, err
}
if err := flagConfig.validate(); err != nil { if err := flagConfig.validate(); err != nil {
return nil, err return nil, err
} }
@@ -494,84 +596,203 @@ func (c *client) GetAllFlags(flagConfig FeatureFlagPayloadNoKey) (map[string]int
flagConfig.PersonProperties, flagConfig.GroupProperties) flagConfig.PersonProperties, flagConfig.GroupProperties)
} }
// Check context after operation
if ctx.Err() != nil {
return nil, ctx.Err()
}
return flagsValue, err return flagsValue, err
} }
// Close and flush metrics. // Close gracefully shuts down the client, flushing any pending messages.
func (c *client) Close() (err error) { // If ShutdownTimeout is set to a positive duration, Close waits up to that
defer func() { // duration for in-flight requests to complete. Otherwise, it waits indefinitely.
// Always recover, a panic could be raised if `c`.quit was closed which // Close is safe to call multiple times; subsequent calls return ErrClosed.
// means the method was called more than once. func (c *client) Close() error {
if recover() != nil { if c.ShutdownTimeout <= 0 {
err = ErrClosed // Zero or negative timeout means wait indefinitely (backward compatible)
return c.CloseWithContext(context.Background())
} }
}() ctx, cancel := context.WithTimeout(context.Background(), c.ShutdownTimeout)
defer cancel()
return c.CloseWithContext(ctx)
}
// CloseWithContext gracefully shuts down the client with the provided context.
// The context can be used to control the shutdown deadline. If the context
// is cancelled before shutdown completes, in-flight requests may be aborted.
// CloseWithContext is safe to call multiple times; subsequent calls return ErrClosed.
func (c *client) CloseWithContext(ctx context.Context) error {
var err error
alreadyClosed := true
c.closeOnce.Do(func() {
alreadyClosed = false
// Mark as closed to fast-fail new Enqueue calls
c.closed.Store(true)
// Signal the batch loop to stop and drain
close(c.quit) close(c.quit)
// Wait for shutdown with timeout from provided context
select {
case <-c.shutdown:
// Clean shutdown completed
c.debugf("shutdown completed successfully")
case <-ctx.Done():
// Timeout exceeded - cancel client context to abort in-flight requests
c.cancel()
err = fmt.Errorf("shutdown timeout: %w", ctx.Err())
c.Warnf("shutdown timeout exceeded, some messages may be lost")
// Wait for shutdown to acknowledge cancellation
<-c.shutdown <-c.shutdown
return }
})
if alreadyClosed {
return ErrClosed
}
return err
} }
// Asynchronously send a batched requests. // processBatch handles a single batch with guaranteed counter decrement.
func (c *client) sendAsync(msgs []message, wg *sync.WaitGroup, ex *executor) { // It receives the batch from the channel and processes it.
wg.Add(1) func (c *client) processBatch() {
// Receive batch from channel - this also "releases" the semaphore slot
batch, ok := <-c.batches
if !ok {
// Channel closed during shutdown
c.inFlight.Add(-1)
return
}
if !ex.do(func() { // CRITICAL: defer ensures counter decrements on ANY exit path
defer wg.Done() // (success, JSON error, HTTP error, panic, context cancellation)
defer c.inFlight.Add(-1)
// Recover from panics to prevent goroutine death without cleanup
defer func() { defer func() {
// In case a bug is introduced in the send function that triggers
// a panic, we don't want this to ever crash the application so we
// catch it here and log it instead.
if err := recover(); err != nil { if err := recover(); err != nil {
c.Errorf("panic - %s", err) c.Errorf("panic in batch processor: %v", err)
c.notifyFailure(batch.msgs, fmt.Errorf("panic: %v", err))
} }
}() }()
c.send(msgs)
}) { c.send(batch)
wg.Done() }
c.Errorf("sending messages failed - %s", ErrTooManyRequests)
c.notifyFailure(msgs, ErrTooManyRequests) // sendBatch attempts to enqueue a batch for processing.
// Returns true if successful, false if the queue is full after waiting for
// BatchSubmitTimeout (default 100ms). This backpressure smoothing allows
// in-flight requests to complete during transient latency spikes, reducing data loss.
// Set BatchSubmitTimeout to a negative value for non-blocking behavior.
//
// On success, spawns a goroutine to process the batch. The batches channel
// acts as both a queue and concurrency limiter.
func (c *client) sendBatch(batch preparedBatch) bool {
c.inFlight.Add(1)
// Negative timeout = non-blocking (immediate drop when queue is full)
if c.BatchSubmitTimeout < 0 {
select {
case c.batches <- batch:
go c.processBatch()
return true
default:
c.inFlight.Add(-1)
return false
}
}
// Blocking send with timeout - allows in-flight requests to complete during latency spikes
timer := time.NewTimer(c.BatchSubmitTimeout)
select {
case c.batches <- batch:
timer.Stop()
go c.processBatch()
return true
case <-timer.C:
c.inFlight.Add(-1)
return false
} }
} }
// Send batch request. // awaitDrain polls until all in-flight batches complete or context times out.
func (c *client) send(msgs []message) { func (c *client) awaitDrain(ctx context.Context) error {
ticker := time.NewTicker(100 * time.Millisecond)
defer ticker.Stop()
for c.inFlight.Load() > 0 {
select {
case <-ctx.Done():
return ctx.Err()
case <-ticker.C:
// continue polling
}
}
c.debugf("all in-flight batches completed")
return nil
}
// send encodes batch wrapper and performs HTTP POST with retries.
// Messages are already pre-serialized, so only the batch wrapper is serialized here.
func (c *client) send(pb preparedBatch) {
// Build batch with pre-serialized messages - only the wrapper is serialized here
b, err := json.Marshal(batch{ b, err := json.Marshal(batch{
ApiKey: c.key, ApiKey: c.key,
HistoricalMigration: c.HistoricalMigration, HistoricalMigration: c.HistoricalMigration,
Messages: msgs, Messages: pb.data,
}) })
if err != nil { if err != nil {
c.Errorf("marshalling messages - %s", err) c.Errorf("marshalling batch wrapper - %s", err)
c.notifyFailure(msgs, err) c.notifyFailure(pb.msgs, err)
return return
} }
for i := 0; i != c.maxAttempts; i++ { for i := 0; i < c.maxAttempts; i++ {
if err = c.upload(b); err == nil { if err = c.upload(c.ctx, b); err == nil {
c.notifySuccess(msgs) c.notifySuccess(pb.msgs)
return return
} }
// Wait for either a retry timeout or the client to be closed. // Check if context is cancelled (shutdown timeout exceeded)
if c.ctx.Err() != nil {
c.Errorf("%d messages dropped: shutdown timeout", len(pb.msgs))
c.notifyFailure(pb.msgs, err)
return
}
// Wait for retry or shutdown
retryTimer := time.NewTimer(c.RetryAfter(i))
select { select {
case <-time.After(c.RetryAfter(i)): case <-retryTimer.C:
// continue to next attempt
case <-c.quit: case <-c.quit:
c.Errorf("%d messages dropped because they failed to be sent and the client was closed", len(msgs)) // Shutdown initiated - stop timer and exit retry loop
c.notifyFailure(msgs, err) if !retryTimer.Stop() {
<-retryTimer.C // Drain channel if timer already fired
}
c.notifyFailure(pb.msgs, err)
return
case <-c.ctx.Done():
// Context cancelled - stop timer and exit
if !retryTimer.Stop() {
<-retryTimer.C // Drain channel if timer already fired
}
c.notifyFailure(pb.msgs, err)
return return
} }
} }
c.Errorf("%d messages dropped because they failed to be sent after %d attempts", len(msgs), c.maxAttempts) c.Errorf("%d messages dropped after %d attempts", len(pb.msgs), c.maxAttempts)
c.notifyFailure(msgs, err) c.notifyFailure(pb.msgs, err)
} }
// Upload serialized batch message. // Upload serialized batch message.
func (c *client) upload(b []byte) error { func (c *client) upload(ctx context.Context, b []byte) error {
url := c.Endpoint + "/batch/" url := c.Endpoint + "/batch/"
req, err := http.NewRequest("POST", url, bytes.NewReader(b)) req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(b))
if err != nil { if err != nil {
c.Errorf("creating request - %s", err) c.Errorf("creating request - %s", err)
return err return err
@@ -584,7 +805,6 @@ func (c *client) upload(b []byte) error {
req.Header.Add("Content-Length", fmt.Sprintf("%d", len(b))) req.Header.Add("Content-Length", fmt.Sprintf("%d", len(b)))
res, err := c.http.Do(req) res, err := c.http.Do(req)
if err != nil { if err != nil {
c.Warnf("sending request - %s", err) c.Warnf("sending request - %s", err)
return err return err
@@ -603,7 +823,7 @@ func (c *client) report(res *http.Response) (err error) {
return return
} }
if body, err = ioutil.ReadAll(res.Body); err != nil { if body, err = io.ReadAll(res.Body); err != nil {
c.Errorf("response %d %s - %s", res.StatusCode, res.Status, err) c.Errorf("response %d %s - %s", res.StatusCode, res.Status, err)
return return
} }
@@ -612,77 +832,111 @@ func (c *client) report(res *http.Response) (err error) {
return fmt.Errorf("%d %s", res.StatusCode, res.Status) return fmt.Errorf("%d %s", res.StatusCode, res.Status)
} }
// Batch loop. // loop processes messages from the msgs channel, batches them by size,
// and spawns goroutines to send batches.
func (c *client) loop() { func (c *client) loop() {
defer close(c.batches) // prevent any pending receives from blocking
defer close(c.shutdown) defer close(c.shutdown)
if c.featureFlagsPoller != nil { if c.featureFlagsPoller != nil {
defer c.featureFlagsPoller.shutdownPoller() defer c.featureFlagsPoller.shutdownPoller()
} }
wg := &sync.WaitGroup{} var batchData []json.RawMessage
defer wg.Wait() var batchMsgs []APIMessage
var batchSize int
tick := time.NewTicker(c.Interval) tick := time.NewTicker(c.Interval)
defer tick.Stop() defer tick.Stop()
ex := newExecutor(c.maxConcurrentRequests) // Helper to create and send a batch
defer ex.close() flushBatch := func() bool {
if len(batchData) == 0 {
return true
}
batch := preparedBatch{data: batchData, msgs: batchMsgs}
if !c.sendBatch(batch) {
c.Errorf("sending batch failed - %s", ErrTooManyRequests)
c.notifyFailure(batchMsgs, ErrTooManyRequests)
return false
}
return true
}
mq := messageQueue{ // Helper to reset batch state
maxBatchSize: c.BatchSize, resetBatch := func() {
maxBatchBytes: c.maxBatchBytes(), batchData, batchMsgs, batchSize = nil, nil, 0
}
// Helper to process a single message: validate, batch, flush if needed.
// Returns false if message was rejected (oversized).
processMessage := func(prepared preparedMessage) bool {
msgSize := len(prepared.data)
if msgSize > maxMessageBytes {
c.Errorf("message exceeds maximum size (%d > %d)", msgSize, maxMessageBytes)
c.notifyFailure([]APIMessage{prepared.msg}, ErrMessageTooBig)
return false
}
if batchSize+msgSize > maxBatchBytes && len(batchData) > 0 {
flushBatch()
resetBatch()
}
batchData = append(batchData, prepared.data)
batchMsgs = append(batchMsgs, prepared.msg)
batchSize += msgSize
if len(batchData) >= c.BatchSize {
flushBatch()
resetBatch()
}
return true
} }
for { for {
select { select {
case msg := <-c.msgs: case prepared := <-c.msgs:
c.push(&mq, msg, wg, ex) if !processMessage(prepared) {
continue
}
c.debugf("buffer (%d/%d, %d bytes) %v", len(batchData), c.BatchSize, batchSize, prepared.msg)
case <-tick.C: case <-tick.C:
c.flush(&mq, wg, ex) if len(batchData) > 0 {
c.debugf("interval flush – sending %d messages", len(batchData))
flushBatch()
resetBatch()
}
case <-c.quit: case <-c.quit:
c.debugf("exit requested – draining messages") c.debugf("shutdown requested – draining messages")
// Drain the msg channel, we have to close it first so no more // Close msgs channel to stop accepting new messages
// messages can be pushed and otherwise the loop would never end.
close(c.msgs) close(c.msgs)
for msg := range c.msgs {
c.push(&mq, msg, wg, ex) // Drain remaining messages using same logic as normal processing
for prepared := range c.msgs {
processMessage(prepared)
} }
c.flush(&mq, wg, ex) // Flush any remaining messages
c.debugf("exit") if len(batchData) > 0 {
c.debugf("flushing final batch of %d messages", len(batchData))
flushBatch()
}
// Wait for in-flight batches to complete
if err := c.awaitDrain(c.ctx); err != nil {
c.Warnf("shutdown timeout: %d batches still in flight", c.inFlight.Load())
}
c.debugf("shutdown complete")
return return
} }
} }
} }
func (c *client) push(q *messageQueue, m APIMessage, wg *sync.WaitGroup, ex *executor) {
var msg message
var err error
if msg, err = makeMessage(m, maxMessageBytes); err != nil {
c.Errorf("%s - %v", err, m)
c.notifyFailure([]message{{m, nil}}, err)
return
}
c.debugf("buffer (%d/%d) %v", len(q.pending), c.BatchSize, m)
if msgs := q.push(msg); msgs != nil {
c.debugf("exceeded messages batch limit with batch of %d messages – flushing", len(msgs))
c.sendAsync(msgs, wg, ex)
}
}
func (c *client) flush(q *messageQueue, wg *sync.WaitGroup, ex *executor) {
if msgs := q.flush(); msgs != nil {
c.debugf("flushing %d messages", len(msgs))
c.sendAsync(msgs, wg, ex)
}
}
func (c *client) debugf(format string, args ...interface{}) { func (c *client) debugf(format string, args ...interface{}) {
c.Logger.Debugf(format, args...) c.Logger.Debugf(format, args...)
} }
@@ -695,25 +949,18 @@ func (c *client) Warnf(format string, args ...interface{}) {
c.Logger.Warnf(format, args...) c.Logger.Warnf(format, args...)
} }
func (c *client) maxBatchBytes() int { func (c *client) notifySuccess(msgs []APIMessage) {
b, _ := json.Marshal(batch{
Messages: []message{},
})
return maxBatchBytes - len(b)
}
func (c *client) notifySuccess(msgs []message) {
if c.Callback != nil { if c.Callback != nil {
for _, m := range msgs { for _, m := range msgs {
c.Callback.Success(m.msg) c.Callback.Success(m)
} }
} }
} }
func (c *client) notifyFailure(msgs []message, err error) { func (c *client) notifyFailure(msgs []APIMessage, err error) {
if c.Callback != nil { if c.Callback != nil {
for _, m := range msgs { for _, m := range msgs {
c.Callback.Failure(m.msg, err) c.Callback.Failure(m, err)
} }
} }
} }
+52 -30
View File
@@ -2,11 +2,10 @@ package posthog
import ( import (
"bytes" "bytes"
"encoding/json"
"errors" "errors"
"fmt" "fmt"
"io" "io"
"io/ioutil"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"net/url" "net/url"
@@ -14,9 +13,12 @@ import (
"path/filepath" "path/filepath"
"reflect" "reflect"
"strings" "strings"
"sync"
"sync/atomic"
"testing" "testing"
"time" "time"
json "github.com/goccy/go-json"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@@ -110,7 +112,7 @@ var (
Proto: r.Proto, Proto: r.Proto,
ProtoMajor: r.ProtoMajor, ProtoMajor: r.ProtoMajor,
ProtoMinor: r.ProtoMinor, ProtoMinor: r.ProtoMinor,
Body: ioutil.NopCloser(strings.NewReader("")), Body: io.NopCloser(strings.NewReader("")),
Request: r, Request: r,
}, nil }, nil
}) })
@@ -126,7 +128,7 @@ var (
Proto: r.Proto, Proto: r.Proto,
ProtoMajor: r.ProtoMajor, ProtoMajor: r.ProtoMajor,
ProtoMinor: r.ProtoMinor, ProtoMinor: r.ProtoMinor,
Body: ioutil.NopCloser(strings.NewReader(body)), Body: io.NopCloser(strings.NewReader(body)),
Request: r, Request: r,
}, nil }, nil
}) })
@@ -145,7 +147,7 @@ var (
Proto: r.Proto, Proto: r.Proto,
ProtoMajor: r.ProtoMajor, ProtoMajor: r.ProtoMajor,
ProtoMinor: r.ProtoMinor, ProtoMinor: r.ProtoMinor,
Body: ioutil.NopCloser(strings.NewReader("")), Body: io.NopCloser(strings.NewReader("")),
Request: r, Request: r,
}, nil }, nil
}) })
@@ -158,7 +160,7 @@ var (
Proto: r.Proto, Proto: r.Proto,
ProtoMajor: r.ProtoMajor, ProtoMajor: r.ProtoMajor,
ProtoMinor: r.ProtoMinor, ProtoMinor: r.ProtoMinor,
Body: ioutil.NopCloser(readFunc(func(b []byte) (int, error) { return 0, testError })), Body: io.NopCloser(readFunc(func(b []byte) (int, error) { return 0, testError })),
Request: r, Request: r,
}, nil }, nil
}) })
@@ -175,7 +177,7 @@ func fixture(name string) string {
panic(err) panic(err)
} }
defer f.Close() defer f.Close()
b, err := ioutil.ReadAll(f) b, err := io.ReadAll(f)
if err != nil { if err != nil {
panic(err) panic(err)
} }
@@ -726,6 +728,7 @@ func TestClientErrorWithMalformedEndpoint(t *testing.T) {
} }
func TestClientRoundTripperError(t *testing.T) { func TestClientRoundTripperError(t *testing.T) {
t.Parallel()
errchan := make(chan error, 1) errchan := make(chan error, 1)
client, _ := NewWithConfig("0123456789", Config{ client, _ := NewWithConfig("0123456789", Config{
@@ -787,6 +790,7 @@ func TestClientRetryError(t *testing.T) {
} }
func TestClientResponse400(t *testing.T) { func TestClientResponse400(t *testing.T) {
t.Parallel()
errchan := make(chan error, 1) errchan := make(chan error, 1)
client, _ := NewWithConfig("0123456789", Config{ client, _ := NewWithConfig("0123456789", Config{
@@ -808,6 +812,7 @@ func TestClientResponse400(t *testing.T) {
} }
func TestClientResponseBodyError(t *testing.T) { func TestClientResponseBodyError(t *testing.T) {
t.Parallel()
errchan := make(chan error, 1) errchan := make(chan error, 1)
client, _ := NewWithConfig("0123456789", Config{ client, _ := NewWithConfig("0123456789", Config{
@@ -831,38 +836,29 @@ func TestClientResponseBodyError(t *testing.T) {
} }
} }
func TestClientMaxConcurrentRequests(t *testing.T) { func TestClientWorkerPool(t *testing.T) {
reschan := make(chan bool, 1) successCount := atomic.Int32{}
errchan := make(chan error, 1)
client, err := NewWithConfig("0123456789", Config{ client, err := NewWithConfig("0123456789", Config{
Logger: testLogger{t.Logf, t.Logf}, Logger: testLogger{t.Logf, t.Logf},
Callback: testCallback{ Callback: testCallback{
func(m APIMessage) { reschan <- true }, func(m APIMessage) { successCount.Add(1) },
func(m APIMessage, e error) { errchan <- e }, func(m APIMessage, e error) { t.Errorf("unexpected failure: %v", e) },
}, },
Transport: testTransportDelayed, Transport: testTransportDelayed,
// Only one concurrency request can be submitted, because the transport // Use single worker with small batch size to test sequential processing
// introduces a short delay one of the uploads should fail.
BatchSize: 1, BatchSize: 1,
maxConcurrentRequests: 1, MaxEnqueuedRequests: 1,
}) })
require.NoError(t, err) require.NoError(t, err)
// Both messages should succeed with the worker pool (they queue up)
require.NoError(t, client.Enqueue(Capture{DistinctId: "A", Event: "B"})) require.NoError(t, client.Enqueue(Capture{DistinctId: "A", Event: "B"}))
require.NoError(t, client.Enqueue(Capture{DistinctId: "A", Event: "B"})) require.NoError(t, client.Enqueue(Capture{DistinctId: "A", Event: "B"}))
require.NoError(t, client.Close()) require.NoError(t, client.Close())
close(reschan)
close(errchan)
if _, ok := <-reschan; !ok { if successCount.Load() != 2 {
t.Error("one of the requests should have succeeded but the result channel was empty") t.Errorf("expected 2 successful callbacks, got %d", successCount.Load())
}
if err := <-errchan; err == nil {
t.Error("failure callback not triggered after reaching the request limit")
} else if err != ErrTooManyRequests {
t.Errorf("invalid error returned by erroring response body: %T: %s", err, err)
} }
} }
@@ -1130,7 +1126,7 @@ func TestGetFeatureFlagPayloadWithNoPersonalApiKey(t *testing.T) {
} }
// Check request body // Check request body
body, _ := ioutil.ReadAll(r.Body) body, _ := io.ReadAll(r.Body)
var requestData FlagsRequestData var requestData FlagsRequestData
json.Unmarshal(body, &requestData) json.Unmarshal(body, &requestData)
if requestData.DistinctId != tt.flagConfig.DistinctId { if requestData.DistinctId != tt.flagConfig.DistinctId {
@@ -1179,11 +1175,27 @@ func TestGetFeatureFlagWithNoPersonalApiKey(t *testing.T) {
})) }))
defer server.Close() defer server.Close()
// Capture events via callback
var capturedEvent *CaptureInApi
var mu sync.Mutex
eventCaptured := make(chan struct{}, 1)
client, _ := NewWithConfig("Csyjlnlun3OzyNJAafdlv", Config{ client, _ := NewWithConfig("Csyjlnlun3OzyNJAafdlv", Config{
Endpoint: server.URL, Endpoint: server.URL,
BatchSize: 1, // Send immediately
Logger: testLogger{t.Logf, t.Logf}, Logger: testLogger{t.Logf, t.Logf},
Callback: testCallback{ Callback: testCallback{
func(m APIMessage) {}, func(m APIMessage) {
if capture, ok := m.(CaptureInApi); ok {
mu.Lock()
capturedEvent = &capture
mu.Unlock()
select {
case eventCaptured <- struct{}{}:
default:
}
}
},
func(m APIMessage, e error) {}, func(m APIMessage, e error) {},
}, },
}) })
@@ -1205,7 +1217,17 @@ func TestGetFeatureFlagWithNoPersonalApiKey(t *testing.T) {
t.Errorf("Expected flag value %v, got: %v", expectedValue, flagValue) t.Errorf("Expected flag value %v, got: %v", expectedValue, flagValue)
} }
lastEvent := client.GetLastCapturedEvent() // Wait for event to be captured
select {
case <-eventCaptured:
case <-time.After(time.Second):
t.Fatal("Timed out waiting for captured event")
}
mu.Lock()
lastEvent := capturedEvent
mu.Unlock()
if lastEvent == nil || lastEvent.Event != "$feature_flag_called" { if lastEvent == nil || lastEvent.Event != "$feature_flag_called" {
t.Errorf("Expected a $feature_flag_called event, got: %v", lastEvent) t.Errorf("Expected a $feature_flag_called event, got: %v", lastEvent)
} }
@@ -1330,7 +1352,7 @@ func TestGetFeatureFlagWithNoPersonalApiKey(t *testing.T) {
} }
// Check request body // Check request body
body, _ := ioutil.ReadAll(r.Body) body, _ := io.ReadAll(r.Body)
var requestData FlagsRequestData var requestData FlagsRequestData
json.Unmarshal(body, &requestData) json.Unmarshal(body, &requestData)
if requestData.DistinctId != tt.flagConfig.DistinctId { if requestData.DistinctId != tt.flagConfig.DistinctId {
@@ -1467,7 +1489,7 @@ func TestGetAllFeatureFlagsWithNoPersonalApiKey(t *testing.T) {
} }
// Check request body // Check request body
body, _ := ioutil.ReadAll(r.Body) body, _ := io.ReadAll(r.Body)
var requestData FlagsRequestData var requestData FlagsRequestData
json.Unmarshal(body, &requestData) json.Unmarshal(body, &requestData)
if requestData.DistinctId != tt.flagConfig.DistinctId { if requestData.DistinctId != tt.flagConfig.DistinctId {
+496
View File
@@ -0,0 +1,496 @@
package posthog
import (
"fmt"
json "github.com/goccy/go-json"
"io"
"net/http"
"net/http/httptest"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/require"
)
// TestStress_EndToEndThroughput measures end-to-end delivery at various loads
func TestStress_EndToEndThroughput(t *testing.T) {
eventCounts := []int{100, 250, 500}
concurrencyLevels := []int{1, 10, 100, 500, 1000}
for _, eventCount := range eventCounts {
for _, concurrency := range concurrencyLevels {
t.Run(fmt.Sprintf("events_%d_goroutines_%d", eventCount, concurrency), func(t *testing.T) {
// PRE-GENERATE all events with realistic cardinality distribution
pool := NewEventPoolWithCardinalityDistribution(eventCount)
events := pool.Events()
var received atomic.Int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var b batch
json.NewDecoder(r.Body).Decode(&b)
received.Add(int64(len(b.Messages)))
w.WriteHeader(200)
}))
defer server.Close()
client, err := NewWithConfig("test-key", Config{
Endpoint: server.URL,
// Uses production defaults for BatchSize, MaxEnqueuedRequests
})
require.NoError(t, err)
start := time.Now()
// Distribute events across goroutines
var wg sync.WaitGroup
var errorCount atomic.Int64
eventsPerGoroutine := eventCount / concurrency
if eventsPerGoroutine == 0 {
eventsPerGoroutine = 1
}
actualConcurrency := concurrency
if actualConcurrency > eventCount {
actualConcurrency = eventCount
}
for g := 0; g < actualConcurrency && g*eventsPerGoroutine < eventCount; g++ {
wg.Add(1)
startIdx := g * eventsPerGoroutine
endIdx := (g + 1) * eventsPerGoroutine
if endIdx > eventCount {
endIdx = eventCount
}
// Handle remainder for last goroutine
if g == actualConcurrency-1 {
endIdx = eventCount
}
go func(start, end int) {
defer wg.Done()
for i := start; i < end; i++ {
if err := client.Enqueue(events[i]); err != nil {
errorCount.Add(1)
}
}
}(startIdx, endIdx)
}
wg.Wait()
client.Close()
elapsed := time.Since(start)
throughput := float64(eventCount) / elapsed.Seconds()
t.Logf("Throughput: %.2f events/sec, Total time: %v", throughput, elapsed)
require.Equal(t, int64(0), errorCount.Load(),
"No enqueue errors should occur")
require.Equal(t, int64(eventCount), received.Load(),
"All events should be delivered")
})
}
}
}
// TestStress_BatchSizeBoundaries tests behavior at batch size limits
func TestStress_BatchSizeBoundaries(t *testing.T) {
t.Parallel()
// Test batch sizes around common boundaries (50, 250 are defaults)
batchSizes := []int{49, 50, 51, 100, 249, 250, 251, 500}
for _, batchSize := range batchSizes {
t.Run(fmt.Sprintf("batch_%d", batchSize), func(t *testing.T) {
// Use enough events to force multiple batches (at least 3x batch size, minimum 300)
eventCount := batchSize * 3
if eventCount < 300 {
eventCount = 300
}
pool := NewEventPoolWithCardinalityDistribution(eventCount)
var batchesReceived atomic.Int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
batchesReceived.Add(1)
w.WriteHeader(200)
}))
defer server.Close()
client, err := NewWithConfig("test-key", Config{
Endpoint: server.URL,
BatchSize: batchSize,
})
require.NoError(t, err)
var errorCount int
for _, e := range pool.Events() {
if err := client.Enqueue(e); err != nil {
errorCount++
}
}
client.Close()
require.Equal(t, 0, errorCount, "No enqueue errors should occur")
expectedBatches := (eventCount + batchSize - 1) / batchSize
// Allow for some variance due to byte-based batching
minExpected := expectedBatches / 2
if minExpected < 2 {
minExpected = 2
}
if batchesReceived.Load() < int64(minExpected) {
t.Errorf("Expected at least %d batches (ideal: %d), got %d",
minExpected, expectedBatches, batchesReceived.Load())
}
})
}
}
// TestStress_CardinalityDistribution verifies system handles various property cardinalities
func TestStress_CardinalityDistribution(t *testing.T) {
cardinalities := []struct {
name string
cardinality PropertyCardinality
}{
{"low_10-100_props", CardinalityLow},
{"medium_500-2000_props", CardinalityMedium},
{"high_2000-10000_props", CardinalityHigh},
}
eventCount := 200
concurrency := 50
for _, tc := range cardinalities {
t.Run(tc.name, func(t *testing.T) {
// Pre-generate events with specific cardinality
events := make([]Capture, eventCount)
for i := range events {
events[i] = Capture{
DistinctId: fmt.Sprintf("user_%d", i),
Event: realisticEvents[i%len(realisticEvents)],
Timestamp: time.Now(),
Properties: generatePropertiesWithCardinality(i, tc.cardinality),
Groups: generateGroupsWithCardinality(i, tc.cardinality),
}
}
var received atomic.Int64
var totalBytes atomic.Int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, _ := io.ReadAll(r.Body)
totalBytes.Add(int64(len(body)))
var b batch
json.Unmarshal(body, &b)
received.Add(int64(len(b.Messages)))
w.WriteHeader(200)
}))
defer server.Close()
client, err := NewWithConfig("test-key", Config{
Endpoint: server.URL,
// Uses production defaults for BatchSize, MaxEnqueuedRequests
})
require.NoError(t, err)
start := time.Now()
var wg sync.WaitGroup
var errorCount atomic.Int64
eventsPerGoroutine := eventCount / concurrency
for g := 0; g < concurrency; g++ {
wg.Add(1)
startIdx := g * eventsPerGoroutine
endIdx := (g + 1) * eventsPerGoroutine
if g == concurrency-1 {
endIdx = eventCount
}
go func(start, end int) {
defer wg.Done()
for i := start; i < end; i++ {
if err := client.Enqueue(events[i]); err != nil {
errorCount.Add(1)
}
}
}(startIdx, endIdx)
}
wg.Wait()
client.Close()
elapsed := time.Since(start)
throughput := float64(eventCount) / elapsed.Seconds()
avgBytesPerEvent := float64(totalBytes.Load()) / float64(received.Load())
t.Logf("Cardinality: %s, Throughput: %.2f events/sec, Avg bytes/event: %.0f",
tc.name, throughput, avgBytesPerEvent)
require.Equal(t, int64(0), errorCount.Load(),
"No enqueue errors should occur")
require.Equal(t, int64(eventCount), received.Load(),
"All events should be delivered")
})
}
}
// TestStress_HighConcurrencyLowVolume tests many goroutines with few events each
func TestStress_HighConcurrencyLowVolume(t *testing.T) {
goroutines := 1000
eventsPerGoroutine := 5
totalEvents := goroutines * eventsPerGoroutine
pool := NewEventPool(totalEvents)
var received atomic.Int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var b batch
json.NewDecoder(r.Body).Decode(&b)
received.Add(int64(len(b.Messages)))
w.WriteHeader(200)
}))
defer server.Close()
client, err := NewWithConfig("test-key", Config{
Endpoint: server.URL,
// Uses production defaults for BatchSize, MaxEnqueuedRequests
})
require.NoError(t, err)
start := time.Now()
var wg sync.WaitGroup
var errorCount atomic.Int64
for g := 0; g < goroutines; g++ {
wg.Add(1)
go func(goroutineID int) {
defer wg.Done()
for i := 0; i < eventsPerGoroutine; i++ {
idx := goroutineID*eventsPerGoroutine + i
if err := client.Enqueue(pool.Get(idx)); err != nil {
errorCount.Add(1)
}
}
}(g)
}
wg.Wait()
client.Close()
elapsed := time.Since(start)
throughput := float64(totalEvents) / elapsed.Seconds()
t.Logf("High concurrency: %d goroutines, %d events, %.2f events/sec, %v total",
goroutines, totalEvents, throughput, elapsed)
require.Equal(t, int64(0), errorCount.Load(),
"No enqueue errors should occur")
require.Equal(t, int64(totalEvents), received.Load(),
"All events should be delivered")
}
// TestStress_LowConcurrencyHighVolume tests few goroutines with many events each
func TestStress_LowConcurrencyHighVolume(t *testing.T) {
goroutines := 4
eventsPerGoroutine := 125
totalEvents := goroutines * eventsPerGoroutine
pool := NewEventPool(totalEvents)
var received atomic.Int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var b batch
json.NewDecoder(r.Body).Decode(&b)
received.Add(int64(len(b.Messages)))
w.WriteHeader(200)
}))
defer server.Close()
client, err := NewWithConfig("test-key", Config{
Endpoint: server.URL,
// Uses production defaults for BatchSize, MaxEnqueuedRequests
})
require.NoError(t, err)
start := time.Now()
var wg sync.WaitGroup
var errorCount atomic.Int64
for g := 0; g < goroutines; g++ {
wg.Add(1)
go func(goroutineID int) {
defer wg.Done()
for i := 0; i < eventsPerGoroutine; i++ {
idx := goroutineID*eventsPerGoroutine + i
if err := client.Enqueue(pool.Get(idx)); err != nil {
errorCount.Add(1)
}
}
}(g)
}
wg.Wait()
client.Close()
elapsed := time.Since(start)
throughput := float64(totalEvents) / elapsed.Seconds()
t.Logf("Low concurrency: %d goroutines, %d events, %.2f events/sec, %v total",
goroutines, totalEvents, throughput, elapsed)
require.Equal(t, int64(0), errorCount.Load(),
"No enqueue errors should occur")
require.Equal(t, int64(totalEvents), received.Load(),
"All events should be delivered")
}
// TestStress_MixedCardinality tests a mix of cardinalities in a single test
func TestStress_MixedCardinality(t *testing.T) {
eventCount := 300
concurrency := 30
// Generate events using the realistic distribution
pool := NewEventPoolWithCardinalityDistribution(eventCount)
var received atomic.Int64
var totalBytes atomic.Int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, _ := io.ReadAll(r.Body)
totalBytes.Add(int64(len(body)))
var b batch
json.Unmarshal(body, &b)
received.Add(int64(len(b.Messages)))
w.WriteHeader(200)
}))
defer server.Close()
client, err := NewWithConfig("test-key", Config{
Endpoint: server.URL,
// Uses production defaults for BatchSize, MaxEnqueuedRequests
})
require.NoError(t, err)
start := time.Now()
var wg sync.WaitGroup
var errorCount atomic.Int64
eventsPerGoroutine := eventCount / concurrency
for g := 0; g < concurrency; g++ {
wg.Add(1)
startIdx := g * eventsPerGoroutine
endIdx := (g + 1) * eventsPerGoroutine
if g == concurrency-1 {
endIdx = eventCount
}
go func(start, end int) {
defer wg.Done()
for i := start; i < end; i++ {
if err := client.Enqueue(pool.Get(i)); err != nil {
errorCount.Add(1)
}
}
}(startIdx, endIdx)
}
wg.Wait()
client.Close()
elapsed := time.Since(start)
throughput := float64(eventCount) / elapsed.Seconds()
avgBytesPerEvent := float64(0)
if received.Load() > 0 {
avgBytesPerEvent = float64(totalBytes.Load()) / float64(received.Load())
}
t.Logf("Mixed cardinality: %d events, %.2f events/sec, %.0f avg bytes/event",
eventCount, throughput, avgBytesPerEvent)
require.Equal(t, int64(0), errorCount.Load(),
"No enqueue errors should occur")
require.Equal(t, int64(eventCount), received.Load(),
"All events should be delivered")
}
// TestStress_RapidCloseReopen tests rapid client close and reopen
func TestStress_RapidCloseReopen(t *testing.T) {
t.Parallel()
iterations := 10
eventsPerIteration := 50
pool := NewEventPool(eventsPerIteration * iterations)
var totalReceived atomic.Int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var b batch
json.NewDecoder(r.Body).Decode(&b)
totalReceived.Add(int64(len(b.Messages)))
w.WriteHeader(200)
}))
defer server.Close()
var totalErrors int
for i := 0; i < iterations; i++ {
client, err := NewWithConfig("test-key", Config{
Endpoint: server.URL,
BatchSize: 25,
// Uses production defaults for MaxEnqueuedRequests
})
require.NoError(t, err)
for j := 0; j < eventsPerIteration; j++ {
idx := i*eventsPerIteration + j
if err := client.Enqueue(pool.Get(idx)); err != nil {
totalErrors++
}
}
client.Close()
}
require.Equal(t, 0, totalErrors,
"No enqueue errors should occur")
expectedTotal := int64(iterations * eventsPerIteration)
require.Equal(t, expectedTotal, totalReceived.Load(),
"All events across all iterations should be delivered")
}
// TestStress_PrepareForSendUnderLoad verifies prepareForSend works correctly under concurrent load
func TestStress_PrepareForSendUnderLoad(t *testing.T) {
t.Parallel()
cardinalities := []PropertyCardinality{CardinalityLow, CardinalityMedium, CardinalityHigh}
for _, cardinality := range cardinalities {
t.Run(CardinalityName(cardinality), func(t *testing.T) {
pool := NewEventPoolWithCardinality(100, cardinality)
var wg sync.WaitGroup
errorCount := atomic.Int64{}
// Concurrent prepareForSend
for g := 0; g < 10; g++ {
wg.Add(1)
go func() {
defer wg.Done()
for i := 0; i < 100; i++ {
capture := pool.Get(i)
capture.Type = "capture"
data, apiMsg, err := prepareForSend(capture)
if err != nil {
errorCount.Add(1)
}
if len(data) <= 0 {
errorCount.Add(1)
}
if apiMsg == nil {
errorCount.Add(1)
}
}
}()
}
wg.Wait()
require.Equal(t, int64(0), errorCount.Load(),
"prepareForSend should never return error, non-positive size, or nil message")
})
}
}
+391
View File
@@ -0,0 +1,391 @@
package posthog
import (
"crypto/rand"
"encoding/hex"
"fmt"
"strings"
"sync/atomic"
"time"
)
// PropertyCardinality defines the number of properties to generate
type PropertyCardinality int
const (
CardinalityLow PropertyCardinality = iota // 10-100 properties
CardinalityMedium // 500-2000 properties
CardinalityHigh // 2000-10000 properties
)
// Pool sizes per cardinality - balance setup time vs. cycling frequency
const (
LowCardinalityPoolSize = 50000 // Fast to create, ~5s
MediumCardinalityPoolSize = 10000 // Moderate setup time
HighCardinalityPoolSize = 2000 // Expensive, keep small
NumTemplates = 50 // Number of unique property templates per pool
)
// PropertyCardinalityDistribution defines realistic distribution of property counts
// Based on typical PostHog usage patterns
var PropertyCardinalityDistribution = []struct {
Cardinality PropertyCardinality
Weight int // percentage of events with this cardinality
}{
{CardinalityLow, 60}, // 60% of events have 10-100 props
{CardinalityMedium, 30}, // 30% have 500-2000 props
{CardinalityHigh, 10}, // 10% have 2000-10000 props (power users)
}
// EventPool holds pre-generated events to avoid generation time during benchmarks
type EventPool struct {
events []Capture
index atomic.Int64
}
// NewEventPool pre-generates n varied events for benchmark use
// Events are varied enough to avoid caching effects
func NewEventPool(n int) *EventPool {
pool := &EventPool{events: make([]Capture, n)}
for i := 0; i < n; i++ {
pool.events[i] = generateVariedCapture(i)
}
return pool
}
// NewEventPoolWithCardinalityDistribution creates events with realistic cardinality distribution
// Uses template-based generation for efficiency
func NewEventPoolWithCardinalityDistribution(n int) *EventPool {
// Create templates for each cardinality level
templatesPerCardinality := make(map[PropertyCardinality][]templateData)
for _, cardinality := range []PropertyCardinality{CardinalityLow, CardinalityMedium, CardinalityHigh} {
templates := make([]templateData, NumTemplates)
for i := 0; i < NumTemplates; i++ {
templates[i] = templateData{
properties: generatePropertiesWithCardinality(i, cardinality),
groups: generateGroupsWithCardinality(i, cardinality),
}
}
templatesPerCardinality[cardinality] = templates
}
// Clone templates to fill pool - each slot gets unique maps
pool := &EventPool{events: make([]Capture, n)}
for i := 0; i < n; i++ {
cardinality := selectCardinality(i)
templates := templatesPerCardinality[cardinality]
tmpl := templates[i%len(templates)]
pool.events[i] = Capture{
DistinctId: generateDistinctId(i),
Event: realisticEvents[i%len(realisticEvents)],
Timestamp: time.Now().Add(time.Duration(-i) * time.Second),
Properties: cloneProperties(tmpl.properties),
Groups: cloneGroups(tmpl.groups),
}
}
return pool
}
// NewEventPoolWithCardinality creates all events with a specific cardinality
// Uses template-based generation: creates NumTemplates unique property sets,
// then clones them to fill the pool. Each slot gets unique maps to prevent races.
func NewEventPoolWithCardinality(n int, cardinality PropertyCardinality) *EventPool {
// Create templates with varied properties
templates := make([]templateData, NumTemplates)
for i := 0; i < NumTemplates; i++ {
templates[i] = templateData{
properties: generatePropertiesWithCardinality(i, cardinality),
groups: generateGroupsWithCardinality(i, cardinality),
}
}
// Clone templates to fill pool - each slot gets unique maps
pool := &EventPool{events: make([]Capture, n)}
for i := 0; i < n; i++ {
tmpl := templates[i%NumTemplates]
pool.events[i] = Capture{
DistinctId: generateDistinctId(i),
Event: realisticEvents[i%len(realisticEvents)],
Timestamp: time.Now().Add(time.Duration(-i) * time.Second),
Properties: cloneProperties(tmpl.properties),
Groups: cloneGroups(tmpl.groups),
}
}
return pool
}
// NewEventPoolWithDefaultSize creates a pool with cardinality-appropriate default size
// Use this when you don't need a specific pool size
func NewEventPoolWithDefaultSize(cardinality PropertyCardinality) *EventPool {
size := defaultPoolSize(cardinality)
return NewEventPoolWithCardinality(size, cardinality)
}
// defaultPoolSize returns the recommended pool size for a cardinality level
func defaultPoolSize(cardinality PropertyCardinality) int {
switch cardinality {
case CardinalityLow:
return LowCardinalityPoolSize
case CardinalityMedium:
return MediumCardinalityPoolSize
case CardinalityHigh:
return HighCardinalityPoolSize
default:
return LowCardinalityPoolSize
}
}
// templateData holds pre-generated properties/groups for cloning
type templateData struct {
properties Properties
groups Groups
}
// cloneProperties creates a shallow copy of a Properties map
func cloneProperties(src Properties) Properties {
if src == nil {
return nil
}
dst := make(Properties, len(src))
for k, v := range src {
dst[k] = v
}
return dst
}
// cloneGroups creates a shallow copy of a Groups map
func cloneGroups(src Groups) Groups {
if src == nil {
return nil
}
dst := make(Groups, len(src))
for k, v := range src {
dst[k] = v
}
return dst
}
// Next returns the next event (cycling through pool) - thread-safe
// Returns a copy with cloned Properties/Groups to prevent races when the SDK
// modifies the maps (via Merge, prepareForSend) in the Enqueue caller thread
func (p *EventPool) Next() Capture {
idx := p.index.Add(1) - 1
original := p.events[idx%int64(len(p.events))]
return Capture{
DistinctId: original.DistinctId,
Event: original.Event,
Timestamp: original.Timestamp,
Properties: cloneProperties(original.Properties),
Groups: cloneGroups(original.Groups),
}
}
// Get returns the event at index i (cycling through pool)
func (p *EventPool) Get(i int) Capture {
return p.events[i%len(p.events)]
}
// Len returns the number of events in the pool
func (p *EventPool) Len() int {
return len(p.events)
}
// Events returns the underlying slice of events
func (p *EventPool) Events() []Capture {
return p.events
}
// selectCardinality returns cardinality based on weighted distribution
func selectCardinality(seed int) PropertyCardinality {
r := seed % 100
cumulative := 0
for _, dist := range PropertyCardinalityDistribution {
cumulative += dist.Weight
if r < cumulative {
return dist.Cardinality
}
}
return CardinalityLow
}
// generateVariedCapture creates a unique, realistic event
func generateVariedCapture(seed int) Capture {
return Capture{
DistinctId: generateDistinctId(seed),
Event: realisticEvents[seed%len(realisticEvents)],
Timestamp: time.Now().Add(time.Duration(-seed) * time.Second),
Properties: generateVariedProperties(seed),
}
}
// generateDistinctId creates varied distinct IDs to avoid caching
func generateDistinctId(seed int) string {
formats := []func(int) string{
func(s int) string { return fmt.Sprintf("user_%[email protected]", s) },
func(s int) string { return fmt.Sprintf("auth0|%012d", s) },
func(s int) string { return generateUUID(s) },
func(s int) string { return fmt.Sprintf("anon_%s", randomHex(8)) },
}
return formats[seed%len(formats)](seed)
}
// generateUUID creates a deterministic UUID-like string based on seed
func generateUUID(seed int) string {
return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x",
seed,
(seed>>8)&0xffff,
(seed>>16)&0xffff,
(seed>>24)&0xffff,
seed*1000000)
}
// randomHex generates a random hex string of specified length
func randomHex(length int) string {
bytes := make([]byte, length/2+1)
rand.Read(bytes)
return hex.EncodeToString(bytes)[:length]
}
// Realistic event names matching PostHog patterns
var realisticEvents = []string{
"$pageview", "$autocapture", "$identify", "$pageleave",
"user_signed_up", "purchase_completed", "item_added_to_cart",
"feature_flag_called", "$exception", "form_submitted",
"button_clicked", "search_performed", "video_played",
}
// generateVariedProperties creates realistic property payloads with moderate cardinality
func generateVariedProperties(seed int) Properties {
props := Properties{
"$lib": "posthog-go",
"$lib_version": "1.0.0",
"session_id": fmt.Sprintf("sess_%d", seed/100),
"page_url": fmt.Sprintf("https://app.example.com/page/%d", seed%50),
}
// Add varied properties based on seed
if seed%3 == 0 {
props["user_agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)"
props["screen_width"] = 1920
props["screen_height"] = 1080
}
if seed%5 == 0 {
props["utm_source"] = "google"
props["utm_campaign"] = fmt.Sprintf("campaign_%d", seed%10)
}
if seed%7 == 0 {
props["$referrer"] = "https://google.com/search?q=example"
}
return props
}
// generatePropertiesWithCardinality creates properties with specified cardinality
func generatePropertiesWithCardinality(seed int, cardinality PropertyCardinality) Properties {
var propCount int
switch cardinality {
case CardinalityLow:
propCount = 10 + (seed % 91) // 10-100
case CardinalityMedium:
propCount = 500 + (seed % 1501) // 500-2000
case CardinalityHigh:
propCount = 2000 + (seed % 8001) // 2000-10000
}
props := Properties{
"$lib": "posthog-go",
"$lib_version": "1.0.0",
}
// Realistic string patterns to simulate real payloads
stringPatterns := []func(seed, i int) string{
func(s, i int) string { return fmt.Sprintf("string_value_%d_%d", s, i) }, // short
func(s, i int) string { return fmt.Sprintf("https://app.example.com/dashboard/project/%d/page/%d", s%100, i%50) }, // URL
func(s, i int) string { return "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36" }, // user agent
func(s, i int) string { return fmt.Sprintf("user_%d@company%d.example.com", s, i%10) }, // email
func(s, i int) string { return strings.Repeat("x", 50+i%200) }, // varied length
}
for i := 0; i < propCount; i++ {
key := fmt.Sprintf("prop_%d", i)
// Vary value types based on index - 8 types for more realism
switch i % 8 {
case 0, 1, 2: // 3/8 strings with varied patterns
props[key] = stringPatterns[i%len(stringPatterns)](seed, i)
case 3:
props[key] = seed*1000 + i // integer
case 4:
props[key] = float64(seed) + float64(i)/100.0 // float
case 5:
props[key] = i%2 == 0 // bool
case 6:
props[key] = []string{fmt.Sprintf("item_%d", i), fmt.Sprintf("item_%d", i+1)} // array
case 7:
props[key] = map[string]interface{}{"nested_key": i, "nested_val": seed} // nested map
}
}
return props
}
// generateGroupsWithCardinality creates groups with specified cardinality
// Group cardinality is scaled proportionally but capped at realistic limits
func generateGroupsWithCardinality(seed int, cardinality PropertyCardinality) Groups {
var groupCount int
switch cardinality {
case CardinalityLow:
groupCount = 1 + (seed % 3) // 1-3
case CardinalityMedium:
groupCount = 3 + (seed % 5) // 3-7
case CardinalityHigh:
groupCount = 5 + (seed % 6) // 5-10
}
groups := Groups{}
groupTypes := []string{"company", "project", "team", "workspace", "organization", "department", "region", "account", "tenant", "division"}
for i := 0; i < groupCount && i < len(groupTypes); i++ {
groups[groupTypes[i]] = fmt.Sprintf("%s_%d", groupTypes[i], seed+i)
}
return groups
}
// GenerateCapturesBatch generates a batch of captures for stress tests
func GenerateCapturesBatch(count int) []Capture {
pool := NewEventPool(count)
return pool.events
}
// GenerateCapturesBatchWithCardinality generates a batch of captures with specific cardinality
func GenerateCapturesBatchWithCardinality(count int, cardinality PropertyCardinality) []Capture {
pool := NewEventPoolWithCardinality(count, cardinality)
return pool.events
}
// Property edge case generators for testing edge cases
var edgeCaseProperties = map[string]Properties{
"unicode": {"名前": "テスト", "emoji": "🚀", "arabic": "مرحبا"},
"nested": {"user": map[string]interface{}{"id": 1, "meta": map[string]interface{}{"created": "2024-01-01"}}},
"large_string": {"data": strings.Repeat("x", 10000)},
"special_chars": {"path": "/api/v1?foo=bar&baz=qux", "json_in_string": `{"nested": "json"}`},
"numbers": {"int": 42, "float": 3.14159, "negative": -100, "zero": 0},
}
// GetEdgeCaseProperties returns edge case properties for testing
func GetEdgeCaseProperties(name string) Properties {
if props, ok := edgeCaseProperties[name]; ok {
return props
}
return Properties{}
}
// CardinalityName returns a human-readable name for a cardinality level
func CardinalityName(c PropertyCardinality) string {
switch c {
case CardinalityLow:
return "low_10-100_props"
case CardinalityMedium:
return "medium_500-2000_props"
case CardinalityHigh:
return "high_2000-10000_props"
default:
return "unknown"
}
}
+252
View File
@@ -0,0 +1,252 @@
package posthog
import (
"io"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
)
// TestScenario defines common test scenarios for transport mocking
type TestScenario int
const (
ScenarioOK TestScenario = iota
ScenarioDelayed
ScenarioBadRequest
ScenarioBodyError
ScenarioNetworkError
ScenarioServerError
)
// UnifiedCallback is a comprehensive callback mock with channels for synchronization
type UnifiedCallback struct {
t *testing.T
mu sync.Mutex
successCount int
failureCount int
lastError error
SuccessChan chan APIMessage
FailureChan chan error
}
// NewUnifiedCallback creates a new unified callback for testing
func NewUnifiedCallback(t *testing.T) *UnifiedCallback {
return &UnifiedCallback{
t: t,
SuccessChan: make(chan APIMessage, 100),
FailureChan: make(chan error, 100),
}
}
func (c *UnifiedCallback) Success(msg APIMessage) {
c.mu.Lock()
c.successCount++
c.mu.Unlock()
if c.t != nil {
c.t.Logf("Callback: SUCCESS")
}
select {
case c.SuccessChan <- msg:
default:
}
}
func (c *UnifiedCallback) Failure(msg APIMessage, err error) {
c.mu.Lock()
c.failureCount++
c.lastError = err
c.mu.Unlock()
if c.t != nil {
c.t.Logf("Callback: FAILURE - %v", err)
}
select {
case c.FailureChan <- err:
default:
}
}
func (c *UnifiedCallback) GetCounts() (success, failure int) {
c.mu.Lock()
defer c.mu.Unlock()
return c.successCount, c.failureCount
}
func (c *UnifiedCallback) GetLastError() error {
c.mu.Lock()
defer c.mu.Unlock()
return c.lastError
}
// MockServerConfig configures a mock HTTP server for testing
type MockServerConfig struct {
// Response configurations
BatchResponse string
BatchStatusCode int
FlagsResponse string
FlagsStatusCode int
LocalEvalResponse string
LocalEvalStatus int
// Behavior modifiers
LatencyMs int
FailAfterN int
// Handlers for custom behavior
BatchHandler func(body []byte)
FlagsHandler func(w http.ResponseWriter, r *http.Request)
LocalEvalHandler func(w http.ResponseWriter, r *http.Request)
}
// MockServerBuilder builds configurable mock servers
type MockServerBuilder struct {
config MockServerConfig
requestCount int
mu sync.Mutex
}
// NewMockServerBuilder creates a new mock server builder with defaults
func NewMockServerBuilder() *MockServerBuilder {
return &MockServerBuilder{
config: MockServerConfig{
BatchStatusCode: http.StatusOK,
FlagsStatusCode: http.StatusOK,
LocalEvalStatus: http.StatusOK,
},
}
}
func (b *MockServerBuilder) WithBatchResponse(response string, statusCode int) *MockServerBuilder {
b.config.BatchResponse = response
b.config.BatchStatusCode = statusCode
return b
}
func (b *MockServerBuilder) WithFlagsResponse(response string, statusCode int) *MockServerBuilder {
b.config.FlagsResponse = response
b.config.FlagsStatusCode = statusCode
return b
}
func (b *MockServerBuilder) WithLocalEvalResponse(response string, statusCode int) *MockServerBuilder {
b.config.LocalEvalResponse = response
b.config.LocalEvalStatus = statusCode
return b
}
func (b *MockServerBuilder) WithBatchHandler(handler func(body []byte)) *MockServerBuilder {
b.config.BatchHandler = handler
return b
}
func (b *MockServerBuilder) WithFailAfterN(n int) *MockServerBuilder {
b.config.FailAfterN = n
return b
}
func (b *MockServerBuilder) GetRequestCount() int {
b.mu.Lock()
defer b.mu.Unlock()
return b.requestCount
}
func (b *MockServerBuilder) Build() *httptest.Server {
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
b.mu.Lock()
b.requestCount++
count := b.requestCount
b.mu.Unlock()
// Check if we should fail
if b.config.FailAfterN > 0 && count > b.config.FailAfterN {
w.WriteHeader(http.StatusInternalServerError)
return
}
// Route to appropriate handler
switch {
case strings.HasPrefix(r.URL.Path, "/batch"):
if b.config.BatchHandler != nil {
body, _ := io.ReadAll(r.Body)
b.config.BatchHandler(body)
}
w.WriteHeader(b.config.BatchStatusCode)
if b.config.BatchResponse != "" {
w.Write([]byte(b.config.BatchResponse))
}
case strings.HasPrefix(r.URL.Path, "/flags"):
if b.config.FlagsHandler != nil {
b.config.FlagsHandler(w, r)
return
}
w.WriteHeader(b.config.FlagsStatusCode)
if b.config.FlagsResponse != "" {
w.Write([]byte(b.config.FlagsResponse))
}
case strings.HasPrefix(r.URL.Path, "/api/feature_flag/local_evaluation"):
if b.config.LocalEvalHandler != nil {
b.config.LocalEvalHandler(w, r)
return
}
w.WriteHeader(b.config.LocalEvalStatus)
if b.config.LocalEvalResponse != "" {
w.Write([]byte(b.config.LocalEvalResponse))
}
default:
w.WriteHeader(http.StatusNotFound)
}
}))
}
// NewTestTransport creates a transport for the given test scenario
func NewTestTransport(scenario TestScenario) http.RoundTripper {
switch scenario {
case ScenarioOK:
return testTransportOK
case ScenarioDelayed:
return testTransportDelayed
case ScenarioBadRequest:
return testTransportBadRequest
case ScenarioBodyError:
return testTransportBodyError
case ScenarioNetworkError:
return testTransportError
case ScenarioServerError:
return roundTripperFunc(func(r *http.Request) (*http.Response, error) {
return &http.Response{
Status: http.StatusText(http.StatusInternalServerError),
StatusCode: http.StatusInternalServerError,
Proto: r.Proto,
ProtoMajor: r.ProtoMajor,
ProtoMinor: r.ProtoMinor,
Body: io.NopCloser(strings.NewReader("")),
Request: r,
}, nil
})
default:
return testTransportOK
}
}
// NoOpTransport returns a transport that succeeds immediately without any network calls
// Useful for benchmarking enqueue throughput without HTTP overhead
func NoOpTransport() http.RoundTripper {
return roundTripperFunc(func(r *http.Request) (*http.Response, error) {
io.Copy(io.Discard, r.Body)
return &http.Response{
Status: http.StatusText(http.StatusOK),
StatusCode: http.StatusOK,
Proto: r.Proto,
ProtoMajor: r.ProtoMajor,
ProtoMinor: r.ProtoMinor,
Body: io.NopCloser(strings.NewReader("")),
Request: r,
}, nil
})
}