mirror of
https://github.com/hanzoai/traces.git
synced 2026-10-30 18:20:41 +00:00
Martin DisibioandGitHub
0313a02c33
* New analyse block data, more dedicated columns * Add analyse block arg to output quoted/spaced list for other cli command * Committing cli cardinality analysis * Different dedicated columns per scope, turn dictionary on and off dynamically based on column cardinality * Experiments * First version of working RewriteSchema * Simple version * Updates * Update to new options pattern in test branch * Fixes after merging main * Cleanup/simplify analyse block code for blobs * Make blob detection in analyse block optional * Update to same default * Update to final parquet-go StructTag option * Delete unused dedicated columns, update all references of schema/reader/write to account for it * cli cleanup and support blobs in jsonnet * Upgrade parquet-go to get remaining fix * cleanup * Fix event dedicated attr writes. Readd tests and cleanup * cleanup * lint * Support vParquet5 for analyse block cli command * changelog + bump encoding version * Fix analyse blocks combination of data * More test coverage of event dedicated columns * Fix wal block, search, and tag lookup on event dedicated columns. More test coverage * Revert printing scope in simple summary
101 lines
2.8 KiB
Go
101 lines
2.8 KiB
Go
package vparquet5
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
|
|
"github.com/parquet-go/parquet-go"
|
|
|
|
tempo_io "github.com/grafana/tempo/pkg/io"
|
|
"github.com/grafana/tempo/pkg/parquetquery"
|
|
"github.com/grafana/tempo/pkg/util"
|
|
"github.com/grafana/tempo/tempodb/encoding/common"
|
|
)
|
|
|
|
// openForIteration opens and returns the parquet file. Note that this is currently only used for compaction and is tuned for this kind of workload.
|
|
// In particular it uses the tempo_io.BufferedReaderAt which is slower but uses less memory and incurs fewer backend calls.
|
|
func (b *backendBlock) openForIteration(ctx context.Context) (*parquet.File, *parquet.Reader, error) { //nolint:all //deprecated
|
|
rr := NewBackendReaderAt(ctx, b.r, DataFileName, b.meta)
|
|
|
|
// 128 MB memory buffering
|
|
br := tempo_io.NewBufferedReaderAt(rr, int64(b.meta.Size_), 2*1024*1024, 64)
|
|
|
|
sch, _, readerOptions := SchemaWithDynamicChanges(b.meta.DedicatedColumns)
|
|
|
|
o := []parquet.FileOption{
|
|
parquet.SkipBloomFilters(true),
|
|
parquet.SkipPageIndex(true),
|
|
parquet.FileReadMode(parquet.ReadModeAsync),
|
|
parquet.FileSchema(sch),
|
|
}
|
|
|
|
pf, err := parquet.OpenFile(br, int64(b.meta.Size_), o...)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
r := parquet.NewReader(pf, append(readerOptions, sch)...)
|
|
return pf, r, nil
|
|
}
|
|
|
|
func (b *backendBlock) rawIter(ctx context.Context, pool *rowPool) (*rawIterator, error) {
|
|
pf, r, err := b.openForIteration(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
traceIDIndex, _, _ := parquetquery.GetColumnIndexByPath(pf, TraceIDColumnName)
|
|
if traceIDIndex < 0 {
|
|
return nil, fmt.Errorf("cannot find trace ID column in '%s' in block '%s'", TraceIDColumnName, b.meta.BlockID.String())
|
|
}
|
|
|
|
return &rawIterator{b.meta.BlockID.String(), r, traceIDIndex, pool}, nil
|
|
}
|
|
|
|
type rawIterator struct {
|
|
blockID string
|
|
r *parquet.Reader //nolint:all //deprecated
|
|
traceIDIndex int
|
|
pool *rowPool
|
|
}
|
|
|
|
var _ RawIterator = (*rawIterator)(nil)
|
|
|
|
func (i *rawIterator) getTraceID(r parquet.Row) common.ID {
|
|
for _, v := range r {
|
|
if v.Column() == i.traceIDIndex {
|
|
// Important - clone to get a detached copy that lives outside the pool.
|
|
return v.Clone().ByteArray()
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (i *rawIterator) Next(context.Context) (common.ID, parquet.Row, error) {
|
|
rows := []parquet.Row{i.pool.Get()}
|
|
n, err := i.r.ReadRows(rows)
|
|
if n > 0 {
|
|
return i.getTraceID(rows[0]), rows[0], nil
|
|
}
|
|
|
|
if errors.Is(err, io.EOF) {
|
|
i.pool.Put(rows[0])
|
|
return nil, nil, nil
|
|
}
|
|
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("error iterating through block %s: %w", i.blockID, err)
|
|
}
|
|
return nil, nil, nil
|
|
}
|
|
|
|
func (i *rawIterator) peekNextID(context.Context) (common.ID, error) { // nolint:unused // this is required to satisfy the bookmarkIterator interface
|
|
return nil, util.ErrUnsupported
|
|
}
|
|
|
|
func (i *rawIterator) Close() {
|
|
i.r.Close()
|
|
}
|