mirror of
https://github.com/hanzoai/traces.git
synced 2026-10-30 18:20:41 +00:00
Martin DisibioandGitHub
011f1b0fcc
* test * Refactor dupe errors, skip compaction, fix error handling in search * Lint, tests * Fix tests
99 lines
2.7 KiB
Go
99 lines
2.7 KiB
Go
package vparquet4
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
|
|
"github.com/parquet-go/parquet-go"
|
|
|
|
tempo_io "github.com/grafana/tempo/pkg/io"
|
|
"github.com/grafana/tempo/pkg/parquetquery"
|
|
"github.com/grafana/tempo/pkg/util"
|
|
"github.com/grafana/tempo/tempodb/encoding/common"
|
|
)
|
|
|
|
// openForIteration opens and returns the parquet file. Note that this is currently only used for compaction and is tuned for this kind of workload.
|
|
// In particular it uses the tempo_io.BufferedReaderAt which is slower but uses less memory and incurs fewer backend calls.
|
|
func (b *backendBlock) openForIteration(ctx context.Context) (*parquet.File, *parquet.Reader, error) { //nolint:all //deprecated
|
|
rr := NewBackendReaderAt(ctx, b.r, DataFileName, b.meta)
|
|
|
|
// 128 MB memory buffering
|
|
br := tempo_io.NewBufferedReaderAt(rr, int64(b.meta.Size_), 2*1024*1024, 64)
|
|
|
|
o := []parquet.FileOption{
|
|
parquet.SkipBloomFilters(true),
|
|
parquet.SkipPageIndex(true),
|
|
parquet.FileSchema(parquetSchema),
|
|
parquet.FileReadMode(parquet.ReadModeAsync),
|
|
}
|
|
|
|
pf, err := parquet.OpenFile(br, int64(b.meta.Size_), o...)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
r := parquet.NewReader(pf, parquet.SchemaOf(&Trace{}))
|
|
return pf, r, nil
|
|
}
|
|
|
|
func (b *backendBlock) rawIter(ctx context.Context, pool *rowPool) (*rawIterator, error) {
|
|
pf, r, err := b.openForIteration(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
traceIDIndex, _, _ := parquetquery.GetColumnIndexByPath(pf, TraceIDColumnName)
|
|
if traceIDIndex < 0 {
|
|
return nil, fmt.Errorf("cannot find trace ID column in '%s' in block '%s'", TraceIDColumnName, b.meta.BlockID.String())
|
|
}
|
|
|
|
return &rawIterator{b.meta.BlockID.String(), r, traceIDIndex, pool}, nil
|
|
}
|
|
|
|
type rawIterator struct {
|
|
blockID string
|
|
r *parquet.Reader //nolint:all //deprecated
|
|
traceIDIndex int
|
|
pool *rowPool
|
|
}
|
|
|
|
var _ RawIterator = (*rawIterator)(nil)
|
|
|
|
func (i *rawIterator) getTraceID(r parquet.Row) common.ID {
|
|
for _, v := range r {
|
|
if v.Column() == i.traceIDIndex {
|
|
// Important - clone to get a detached copy that lives outside the pool.
|
|
return v.Clone().ByteArray()
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (i *rawIterator) Next(context.Context) (common.ID, parquet.Row, error) {
|
|
rows := []parquet.Row{i.pool.Get()}
|
|
n, err := i.r.ReadRows(rows)
|
|
if n > 0 {
|
|
return i.getTraceID(rows[0]), rows[0], nil
|
|
}
|
|
|
|
if errors.Is(err, io.EOF) {
|
|
i.pool.Put(rows[0])
|
|
return nil, nil, nil
|
|
}
|
|
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("error iterating through block %s: %w", i.blockID, err)
|
|
}
|
|
return nil, nil, nil
|
|
}
|
|
|
|
func (i *rawIterator) peekNextID(context.Context) (common.ID, error) { // nolint:unused // this is required to satisfy the bookmarkIterator interface
|
|
return nil, util.ErrUnsupported
|
|
}
|
|
|
|
func (i *rawIterator) Close() {
|
|
i.r.Close()
|
|
}
|