mirror of
https://github.com/hanzoai/telemetry.git
synced 2026-10-30 20:01:49 +00:00
<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue.
Ex. Adding a feature - Explain what this achieves.-->
#### Description
The problem happens because:
T1: Executes `writer.Close()` and gets descheduled after `z.pool.Put(z)`
but before leaving the `Do` function.
T2: Executes `compressor.Decompress`, get the latest re-added writer
from T1, and overwrites the `once` variable.
T1: Resumes and tries to unlock the overwritten `once` which is not
locked.
This is an UB and everything can happen when we corrupt the memory like
this.
```
fatal error: sync: unlock of unlocked mutex
goroutine 48499 [running]:
internal/sync.fatal({0xa4032ec?, 0x373e910417c0?})
GOROOT/src/runtime/panic.go:1191 +0x18
internal/sync.(*Mutex).unlockSlow(0x373e910417d4, 0xffffffff)
GOROOT/src/internal/sync/mutex.go:204 +0x35
internal/sync.(*Mutex).Unlock(...)
GOROOT/src/internal/sync/mutex.go:198
sync.(*Mutex).Unlock(...)
GOROOT/src/sync/mutex.go:65
sync.(*Once).doSlow(0x373e91c44f50?, 0x373eadc29fed?)
GOROOT/src/sync/once.go:80 +0xe4
sync.(*Once).Do(...)
GOROOT/src/sync/once.go:69
go.opentelemetry.io/collector/internal/grpccompression/snappy.(*reader).Read(0x373e910417c0, {0x373eadc29fed?, 0x373e4d674300?, 0x8000?})
external/io_opentelemetry_go_collector/internal/grpccompression/snappy/snappy.go:96 +0x65
io.(*LimitedReader).Read(0x373eaa8bc6c0, {0x373eadc29fed?, 0x373e4d6526c0?, 0x373eaa8bc6c0?})
GOROOT/src/io/io.go:479 +0x43
google.golang.org/grpc/mem.ReadAll({0xad75ec0, 0x373eaa8bc6c0}, {0xadfedd8, 0x373e4d6526c0})
external/org_golang_google_grpc/mem/buffer_slice.go:268 +0x188
google.golang.org/grpc.decompress({0xae3eea0, 0x373e4d66a320}, {0x373e4d836c60?, 0x373e68b4c8b0?, 0x425745?}, {0x0?, 0x0?}, 0x40000000, {0xadfedd8, 0x373e4d6526c0})
external/org_golang_google_grpc/rpc_util.go:999 +0x15c
google.golang.org/grpc.recvAndDecompress(0x373e9e41f170, {0xad7dfa0, 0x373e96992820}, {0x0, 0x0}, 0x40000000, 0x373e91041d00, {0xae3eea0, 0x373e4d66a320}, 0x1)
external/org_golang_google_grpc/rpc_util.go:947 +0x2ab
google.golang.org/grpc.(*Server).processUnaryRPC(0x373e4f182008, {0xae516e0, 0x373e9e41f0b0}, 0x373e96992820, 0x373e4e86a060, 0x10da6290, 0x0)
external/org_golang_google_grpc/server.go:1386 +0xf25
google.golang.org/grpc.(*Server).handleStream(0x373e4f182008, {0xae53b28, 0x373e4d859d40}, 0x373e96992820)
external/org_golang_google_grpc/server.go:1856 +0xc2e
google.golang.org/grpc.(*Server).serveStreams.func2.1()
external/org_golang_google_grpc/server.go:1065 +0x7f
created by google.golang.org/grpc.(*Server).serveStreams.func2 in goroutine 302
external/org_golang_google_grpc/server.go:1076 +0x11d
```
<!-- Issue number if applicable -->
#### Link to tracking issue
Fixes #15237
Fixes #15320
<!--Describe what testing was performed and which tests were added.-->
#### Testing
<!--Describe the documentation added.-->
#### Documentation
<!--Please delete paragraphs that you did not use before submitting.-->
Signed-off-by: Bogdan Drutu <[email protected]>
179 lines
3.6 KiB
Go
179 lines
3.6 KiB
Go
// Copyright The OpenTelemetry Authors
|
|
// Copyright 2021 gRPC authors.
|
|
// SPDX-License-Identifier: Apache-2.0
|
|
|
|
package snappy
|
|
|
|
import (
|
|
"bytes"
|
|
"io"
|
|
"runtime"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"google.golang.org/grpc/encoding"
|
|
)
|
|
|
|
func TestRegisteredCompression(t *testing.T) {
|
|
for _, tt := range []struct {
|
|
name string
|
|
payload []byte
|
|
}{
|
|
{name: "empty", payload: nil},
|
|
{name: "small", payload: []byte("message request snappy")},
|
|
{name: "large", payload: bytes.Repeat([]byte("snappy payload "), 1024)},
|
|
} {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
comp := newCompressor()
|
|
got := roundTripCompression(t, comp, tt.payload)
|
|
if len(tt.payload) == 0 {
|
|
assert.Empty(t, got)
|
|
return
|
|
}
|
|
assert.Equal(t, tt.payload, got)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestInitRegistration(t *testing.T) {
|
|
comp := encoding.GetCompressor(Name)
|
|
require.NotNil(t, comp)
|
|
assert.Equal(t, Name, comp.Name())
|
|
}
|
|
|
|
func TestRegisterCompressor(t *testing.T) {
|
|
t.Run("registers when missing", func(t *testing.T) {
|
|
var registered encoding.Compressor
|
|
ok := registerCompressor(
|
|
func(string) encoding.Compressor { return nil },
|
|
func(c encoding.Compressor) { registered = c },
|
|
)
|
|
require.True(t, ok)
|
|
require.NotNil(t, registered)
|
|
assert.Equal(t, Name, registered.Name())
|
|
})
|
|
|
|
t.Run("skips when already registered", func(t *testing.T) {
|
|
var called bool
|
|
ok := registerCompressor(
|
|
func(string) encoding.Compressor { return newCompressor() },
|
|
func(encoding.Compressor) { called = true },
|
|
)
|
|
require.False(t, ok)
|
|
assert.False(t, called)
|
|
})
|
|
}
|
|
|
|
func TestConcurrentWriterReuseDoesNotPanic(t *testing.T) {
|
|
prev := runtime.GOMAXPROCS(runtime.NumCPU())
|
|
defer runtime.GOMAXPROCS(prev)
|
|
|
|
comp := newCompressor()
|
|
payload := bytes.Repeat([]byte("snappy payload "), 32)
|
|
|
|
const workers = 6
|
|
const iterations = 100
|
|
|
|
start := make(chan struct{})
|
|
errCh := make(chan error, workers)
|
|
|
|
for range workers {
|
|
go func() {
|
|
<-start
|
|
for range iterations {
|
|
var buf bytes.Buffer
|
|
w, err := comp.Compress(&buf)
|
|
if err != nil {
|
|
errCh <- err
|
|
return
|
|
}
|
|
|
|
if _, err = w.Write(payload); err != nil {
|
|
errCh <- err
|
|
return
|
|
}
|
|
|
|
if err = w.Close(); err != nil {
|
|
errCh <- err
|
|
return
|
|
}
|
|
}
|
|
errCh <- nil
|
|
}()
|
|
}
|
|
|
|
close(start)
|
|
|
|
for range workers {
|
|
require.NoError(t, <-errCh)
|
|
}
|
|
}
|
|
|
|
func TestConcurrentReaderReuseDoesNotPanic(t *testing.T) {
|
|
prev := runtime.GOMAXPROCS(runtime.NumCPU())
|
|
defer runtime.GOMAXPROCS(prev)
|
|
|
|
comp := newCompressor()
|
|
payload := bytes.Repeat([]byte("snappy payload "), 32)
|
|
compressed := compressPayload(t, comp, payload)
|
|
|
|
const workers = 8
|
|
const iterations = 50
|
|
|
|
start := make(chan struct{})
|
|
errCh := make(chan error, workers)
|
|
|
|
for range workers {
|
|
go func() {
|
|
<-start
|
|
for range iterations {
|
|
r, err := comp.Decompress(bytes.NewReader(compressed))
|
|
if err != nil {
|
|
errCh <- err
|
|
return
|
|
}
|
|
|
|
if _, err = io.ReadAll(r); err != nil {
|
|
errCh <- err
|
|
return
|
|
}
|
|
}
|
|
errCh <- nil
|
|
}()
|
|
}
|
|
|
|
close(start)
|
|
|
|
for range workers {
|
|
require.NoError(t, <-errCh)
|
|
}
|
|
}
|
|
|
|
func compressPayload(t *testing.T, comp encoding.Compressor, payload []byte) []byte {
|
|
t.Helper()
|
|
|
|
var buf bytes.Buffer
|
|
w, err := comp.Compress(&buf)
|
|
require.NoError(t, err)
|
|
|
|
_, err = w.Write(payload)
|
|
require.NoError(t, err)
|
|
require.NoError(t, w.Close())
|
|
|
|
return buf.Bytes()
|
|
}
|
|
|
|
func roundTripCompression(t *testing.T, comp encoding.Compressor, payload []byte) []byte {
|
|
t.Helper()
|
|
|
|
compressed := compressPayload(t, comp, payload)
|
|
|
|
r, err := comp.Decompress(bytes.NewReader(compressed))
|
|
require.NoError(t, err)
|
|
|
|
got, err := io.ReadAll(r)
|
|
require.NoError(t, err)
|
|
return got
|
|
}
|