Files
Bogdan DrutuandGitHub ae05bf8c2b Fix memory corruption and fatal error in Snappy (#15321)
<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue.
Ex. Adding a feature - Explain what this achieves.-->
#### Description

The problem happens because:

T1: Executes `writer.Close()` and gets descheduled after `z.pool.Put(z)`
but before leaving the `Do` function.
T2: Executes `compressor.Decompress`, get the latest re-added writer
from T1, and overwrites the `once` variable.
T1: Resumes and tries to unlock the overwritten `once` which is not
locked.

This is an UB and everything can happen when we corrupt the memory like
this.

```
fatal error: sync: unlock of unlocked mutex
goroutine 48499 [running]:
internal/sync.fatal({0xa4032ec?, 0x373e910417c0?})
	GOROOT/src/runtime/panic.go:1191 +0x18
internal/sync.(*Mutex).unlockSlow(0x373e910417d4, 0xffffffff)
	GOROOT/src/internal/sync/mutex.go:204 +0x35
internal/sync.(*Mutex).Unlock(...)
	GOROOT/src/internal/sync/mutex.go:198
sync.(*Mutex).Unlock(...)
	GOROOT/src/sync/mutex.go:65
sync.(*Once).doSlow(0x373e91c44f50?, 0x373eadc29fed?)
	GOROOT/src/sync/once.go:80 +0xe4
sync.(*Once).Do(...)
	GOROOT/src/sync/once.go:69
go.opentelemetry.io/collector/internal/grpccompression/snappy.(*reader).Read(0x373e910417c0, {0x373eadc29fed?, 0x373e4d674300?, 0x8000?})
	external/io_opentelemetry_go_collector/internal/grpccompression/snappy/snappy.go:96 +0x65
io.(*LimitedReader).Read(0x373eaa8bc6c0, {0x373eadc29fed?, 0x373e4d6526c0?, 0x373eaa8bc6c0?})
	GOROOT/src/io/io.go:479 +0x43
google.golang.org/grpc/mem.ReadAll({0xad75ec0, 0x373eaa8bc6c0}, {0xadfedd8, 0x373e4d6526c0})
	external/org_golang_google_grpc/mem/buffer_slice.go:268 +0x188
google.golang.org/grpc.decompress({0xae3eea0, 0x373e4d66a320}, {0x373e4d836c60?, 0x373e68b4c8b0?, 0x425745?}, {0x0?, 0x0?}, 0x40000000, {0xadfedd8, 0x373e4d6526c0})
	external/org_golang_google_grpc/rpc_util.go:999 +0x15c
google.golang.org/grpc.recvAndDecompress(0x373e9e41f170, {0xad7dfa0, 0x373e96992820}, {0x0, 0x0}, 0x40000000, 0x373e91041d00, {0xae3eea0, 0x373e4d66a320}, 0x1)
	external/org_golang_google_grpc/rpc_util.go:947 +0x2ab
google.golang.org/grpc.(*Server).processUnaryRPC(0x373e4f182008, {0xae516e0, 0x373e9e41f0b0}, 0x373e96992820, 0x373e4e86a060, 0x10da6290, 0x0)
	external/org_golang_google_grpc/server.go:1386 +0xf25
google.golang.org/grpc.(*Server).handleStream(0x373e4f182008, {0xae53b28, 0x373e4d859d40}, 0x373e96992820)
	external/org_golang_google_grpc/server.go:1856 +0xc2e
google.golang.org/grpc.(*Server).serveStreams.func2.1()
	external/org_golang_google_grpc/server.go:1065 +0x7f
created by google.golang.org/grpc.(*Server).serveStreams.func2 in goroutine 302
	external/org_golang_google_grpc/server.go:1076 +0x11d
```

<!-- Issue number if applicable -->
#### Link to tracking issue
Fixes #15237 
Fixes #15320 

<!--Describe what testing was performed and which tests were added.-->
#### Testing

<!--Describe the documentation added.-->
#### Documentation

<!--Please delete paragraphs that you did not use before submitting.-->

Signed-off-by: Bogdan Drutu <[email protected]>
2026-05-19 18:24:18 +00:00

179 lines
3.6 KiB
Go

// Copyright The OpenTelemetry Authors
// Copyright 2021 gRPC authors.
// SPDX-License-Identifier: Apache-2.0
package snappy
import (
"bytes"
"io"
"runtime"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/grpc/encoding"
)
func TestRegisteredCompression(t *testing.T) {
for _, tt := range []struct {
name string
payload []byte
}{
{name: "empty", payload: nil},
{name: "small", payload: []byte("message request snappy")},
{name: "large", payload: bytes.Repeat([]byte("snappy payload "), 1024)},
} {
t.Run(tt.name, func(t *testing.T) {
comp := newCompressor()
got := roundTripCompression(t, comp, tt.payload)
if len(tt.payload) == 0 {
assert.Empty(t, got)
return
}
assert.Equal(t, tt.payload, got)
})
}
}
func TestInitRegistration(t *testing.T) {
comp := encoding.GetCompressor(Name)
require.NotNil(t, comp)
assert.Equal(t, Name, comp.Name())
}
func TestRegisterCompressor(t *testing.T) {
t.Run("registers when missing", func(t *testing.T) {
var registered encoding.Compressor
ok := registerCompressor(
func(string) encoding.Compressor { return nil },
func(c encoding.Compressor) { registered = c },
)
require.True(t, ok)
require.NotNil(t, registered)
assert.Equal(t, Name, registered.Name())
})
t.Run("skips when already registered", func(t *testing.T) {
var called bool
ok := registerCompressor(
func(string) encoding.Compressor { return newCompressor() },
func(encoding.Compressor) { called = true },
)
require.False(t, ok)
assert.False(t, called)
})
}
func TestConcurrentWriterReuseDoesNotPanic(t *testing.T) {
prev := runtime.GOMAXPROCS(runtime.NumCPU())
defer runtime.GOMAXPROCS(prev)
comp := newCompressor()
payload := bytes.Repeat([]byte("snappy payload "), 32)
const workers = 6
const iterations = 100
start := make(chan struct{})
errCh := make(chan error, workers)
for range workers {
go func() {
<-start
for range iterations {
var buf bytes.Buffer
w, err := comp.Compress(&buf)
if err != nil {
errCh <- err
return
}
if _, err = w.Write(payload); err != nil {
errCh <- err
return
}
if err = w.Close(); err != nil {
errCh <- err
return
}
}
errCh <- nil
}()
}
close(start)
for range workers {
require.NoError(t, <-errCh)
}
}
func TestConcurrentReaderReuseDoesNotPanic(t *testing.T) {
prev := runtime.GOMAXPROCS(runtime.NumCPU())
defer runtime.GOMAXPROCS(prev)
comp := newCompressor()
payload := bytes.Repeat([]byte("snappy payload "), 32)
compressed := compressPayload(t, comp, payload)
const workers = 8
const iterations = 50
start := make(chan struct{})
errCh := make(chan error, workers)
for range workers {
go func() {
<-start
for range iterations {
r, err := comp.Decompress(bytes.NewReader(compressed))
if err != nil {
errCh <- err
return
}
if _, err = io.ReadAll(r); err != nil {
errCh <- err
return
}
}
errCh <- nil
}()
}
close(start)
for range workers {
require.NoError(t, <-errCh)
}
}
func compressPayload(t *testing.T, comp encoding.Compressor, payload []byte) []byte {
t.Helper()
var buf bytes.Buffer
w, err := comp.Compress(&buf)
require.NoError(t, err)
_, err = w.Write(payload)
require.NoError(t, err)
require.NoError(t, w.Close())
return buf.Bytes()
}
func roundTripCompression(t *testing.T, comp encoding.Compressor, payload []byte) []byte {
t.Helper()
compressed := compressPayload(t, comp, payload)
r, err := comp.Decompress(bytes.NewReader(compressed))
require.NoError(t, err)
got, err := io.ReadAll(r)
require.NoError(t, err)
return got
}