mirror of
https://github.com/hanzoai/traces.git
synced 2026-10-30 18:20:41 +00:00
A. StoewerandGitHub
e181a50c19
* Define default dedicated attribute columns * Use default dedicated columns in block config * vp4 ignores well-known attrs in dedicated columns * vp3 ignores well-known attrs in dedicated columns * Improve tenantindex benchmarks * Use LRU cache for unmarshaled JSON columns * Set version to vParquet5-preview3 * Remove well-known attribute columns on span level * Remove well-known attribute columns on resource level * Convert vp5 test-data and re-enable find by ID test * Generate manifest.md * vp3 and vp4 ignore all unsupported attributes * vp5 ignores unsupported attributes * CHANGELOG.md
85 lines
1.6 KiB
Go
85 lines
1.6 KiB
Go
package backend
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/gogo/protobuf/proto"
|
|
"github.com/klauspost/compress/gzip"
|
|
)
|
|
|
|
const (
|
|
internalFilename = "index.json"
|
|
)
|
|
|
|
var (
|
|
_ proto.Message = (*TenantIndex)(nil)
|
|
Zstd = &ZstdCodec{}
|
|
)
|
|
|
|
func newTenantIndex(meta []*BlockMeta, compactedMeta []*CompactedBlockMeta) *TenantIndex {
|
|
return &TenantIndex{
|
|
CreatedAt: time.Now(),
|
|
Meta: meta,
|
|
CompactedMeta: compactedMeta,
|
|
}
|
|
}
|
|
|
|
// marshal converts to json and compresses the bucketindex
|
|
func (b *TenantIndex) marshal() ([]byte, error) {
|
|
buffer := &bytes.Buffer{}
|
|
|
|
gzip := gzip.NewWriter(buffer)
|
|
gzip.Name = internalFilename
|
|
|
|
jsonBytes, err := json.Marshal(b)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if _, err = gzip.Write(jsonBytes); err != nil {
|
|
return nil, err
|
|
}
|
|
if err = gzip.Flush(); err != nil {
|
|
return nil, err
|
|
}
|
|
if err = gzip.Close(); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return buffer.Bytes(), nil
|
|
}
|
|
|
|
// unmarshal decompresses and unmarshals the results from json
|
|
func (b *TenantIndex) unmarshal(buffer []byte) error {
|
|
gzipReader, err := gzip.NewReader(bytes.NewReader(buffer))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer gzipReader.Close()
|
|
|
|
d := json.NewDecoder(gzipReader)
|
|
return d.Decode(b)
|
|
}
|
|
|
|
func (b *TenantIndex) marshalPb() ([]byte, error) {
|
|
pbBytes, err := proto.Marshal(b)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
buffer := []byte{}
|
|
return Zstd.Encode(pbBytes, buffer)
|
|
}
|
|
|
|
func (b *TenantIndex) unmarshalPb(buffer []byte) error {
|
|
bb, err := Zstd.Decode(buffer)
|
|
if err != nil {
|
|
return fmt.Errorf("error decoding zstd: %w", err)
|
|
}
|
|
|
|
return b.Unmarshal(bb)
|
|
}
|