mirror of
https://github.com/hanzoai/traces.git
synced 2026-10-30 18:20:41 +00:00
049323c40f
* starting parquet support starting parquet support. some things are still todo: compaction, find trace by ID Co-Authored-By: Annanay Agarwal <[email protected]> Co-Authored-By: Martin Disibio <[email protected]> * add parquet reusable streaming block * Add parquet compaction * Add findTraceByID logic and parquet -> proto conversion Signed-off-by: Annanay <[email protected]> * Switch findtracebyid to readrow until we get SeekToRow working * Fix error handling in bufferedReaderAt. Add various instrumentation and log messages * Use jsonpb to roundtrip event attributes * Improve buffered reader when reading near end of file. More instrumentation and tidiness * Add totalBlockBytes instrumentation. Fix inspectedBytes instrumentation * parquet.FindTraceByID check bloom * Make compator flush on trace count until we get the row group size target working * Support maxDuration in parquet search * Correct attribute translation for array/kvlist Signed-off-by: Annanay <[email protected]> * extend roundtrip test Signed-off-by: Annanay <[email protected]> * Use default page buffer size * Fix size url param to be uint64 * Increase parquet searches to 1MB stripes * Add test for findtracebyid * make row group size configurable Signed-off-by: Annanay <[email protected]> * Record parquet footer size in block meta and short-circuit header/footer reads * Add tests for SubstringPredicate * Switch compaction to compact blocks of same version, use re-usable buffer in findtracebyid Signed-off-by: Annanay <[email protected]> * Add search support for remaining well-known attributes and span status, add search tests * Benchmark substringPredicate * Make block version configurable, default to v2 * memoize substringPredicate * Update segmentio/parquet-go to catch SeekToRow fix, revert hack in FindTraceByID Signed-off-by: Annanay <[email protected]> * Add multi block prefetch iterator for use in compactor Signed-off-by: Annanay <[email protected]> * Add BufferedWriter with outgoing queue to flush traces async Signed-off-by: Annanay <[email protected]> Make compactor block complete async Signed-off-by: Annanay <[email protected]> * First pass CLI to dump schema and column sizes from block Signed-off-by: Annanay <[email protected]> * honor search start/end time range * Improve error message, add blockID Signed-off-by: Annanay <[email protected]> * Fix race in InstrumentedPredicate with atomics * Make bufferedReaderAt not single-threaded * Debug SeekToRow panic, add test data from vulture tenant Signed-off-by: Annanay <[email protected]> * fix test * repair tests * Address println warnings * Experiment with bookmark prefetch Signed-off-by: Annanay <[email protected]> Remove quitCh from bookmark Signed-off-by: Annanay <[email protected]> * fix prefetch on the bookmarks Signed-off-by: Annanay <[email protected]> * lint * tracker -> RowNumber, lint * delete unused predicates * lint * lint * mod vendor * remove pool from buffered writer queue Signed-off-by: Annanay <[email protected]> * post merge cleanup Signed-off-by: Annanay <[email protected]> * Close quitCh instead of attempting to signal to it Signed-off-by: Annanay <[email protected]> * split multiblock and prefetch iterators, refactor around interface Signed-off-by: Annanay <[email protected]> * Split search into 2 phases to avoid loading traceid/display columns unless there is a match * Make columnIterator.SeekTo do something * update to catch segmentio/parquet-go#220 Signed-off-by: Annanay <[email protected]> * Fix column index to store entire 32-byte trace ID string. lint * Make parquet honor the adjusted start/end times that come from the wal * post merge cleanup: use compaction metric callbacks Signed-off-by: Annanay <[email protected]> * serverless sum/mods Signed-off-by: Annanay <[email protected]> * parquetquery predicates handle column index not present * Make parquet compaction really honor input block start/end times, add universal test at tempodb-level * parquet skip page indexes while searching to reduce latency on openfile * More testing around iterators and combiner Signed-off-by: Annanay <[email protected]> * Update to latest parquet lib to catch rle bitpack fix * Review feedback: LatestEncoding -> DefaultEncoding * Pass block footerSize through serverless * mod vendor for serverless * Fix tests * add blockID to error message in block iterator Signed-off-by: Annanay <[email protected]> * increase readvalues buffer, add more span instrumentation Signed-off-by: Annanay <[email protected]> * changelog and docs * Remove the use of async writer in vparquet compactor Signed-off-by: Annanay <[email protected]> Co-authored-by: Annanay Agarwal <[email protected]>
94 lines
2.9 KiB
Go
94 lines
2.9 KiB
Go
package brotli
|
|
|
|
/* Copyright 2018 Google Inc. All Rights Reserved.
|
|
|
|
Distributed under MIT license.
|
|
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
*/
|
|
|
|
func (h *hashComposite) HashTypeLength() uint {
|
|
var a uint = h.ha.HashTypeLength()
|
|
var b uint = h.hb.HashTypeLength()
|
|
if a > b {
|
|
return a
|
|
} else {
|
|
return b
|
|
}
|
|
}
|
|
|
|
func (h *hashComposite) StoreLookahead() uint {
|
|
var a uint = h.ha.StoreLookahead()
|
|
var b uint = h.hb.StoreLookahead()
|
|
if a > b {
|
|
return a
|
|
} else {
|
|
return b
|
|
}
|
|
}
|
|
|
|
/* Composite hasher: This hasher allows to combine two other hashers, HASHER_A
|
|
and HASHER_B. */
|
|
type hashComposite struct {
|
|
hasherCommon
|
|
ha hasherHandle
|
|
hb hasherHandle
|
|
params *encoderParams
|
|
}
|
|
|
|
func (h *hashComposite) Initialize(params *encoderParams) {
|
|
h.params = params
|
|
}
|
|
|
|
/* TODO: Initialize of the hashers is defered to Prepare (and params
|
|
remembered here) because we don't get the one_shot and input_size params
|
|
here that are needed to know the memory size of them. Instead provide
|
|
those params to all hashers InitializehashComposite */
|
|
func (h *hashComposite) Prepare(one_shot bool, input_size uint, data []byte) {
|
|
if h.ha == nil {
|
|
var common_a *hasherCommon
|
|
var common_b *hasherCommon
|
|
|
|
common_a = h.ha.Common()
|
|
common_a.params = h.params.hasher
|
|
common_a.is_prepared_ = false
|
|
common_a.dict_num_lookups = 0
|
|
common_a.dict_num_matches = 0
|
|
h.ha.Initialize(h.params)
|
|
|
|
common_b = h.hb.Common()
|
|
common_b.params = h.params.hasher
|
|
common_b.is_prepared_ = false
|
|
common_b.dict_num_lookups = 0
|
|
common_b.dict_num_matches = 0
|
|
h.hb.Initialize(h.params)
|
|
}
|
|
|
|
h.ha.Prepare(one_shot, input_size, data)
|
|
h.hb.Prepare(one_shot, input_size, data)
|
|
}
|
|
|
|
func (h *hashComposite) Store(data []byte, mask uint, ix uint) {
|
|
h.ha.Store(data, mask, ix)
|
|
h.hb.Store(data, mask, ix)
|
|
}
|
|
|
|
func (h *hashComposite) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) {
|
|
h.ha.StoreRange(data, mask, ix_start, ix_end)
|
|
h.hb.StoreRange(data, mask, ix_start, ix_end)
|
|
}
|
|
|
|
func (h *hashComposite) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ring_buffer_mask uint) {
|
|
h.ha.StitchToPreviousBlock(num_bytes, position, ringbuffer, ring_buffer_mask)
|
|
h.hb.StitchToPreviousBlock(num_bytes, position, ringbuffer, ring_buffer_mask)
|
|
}
|
|
|
|
func (h *hashComposite) PrepareDistanceCache(distance_cache []int) {
|
|
h.ha.PrepareDistanceCache(distance_cache)
|
|
h.hb.PrepareDistanceCache(distance_cache)
|
|
}
|
|
|
|
func (h *hashComposite) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) {
|
|
h.ha.FindLongestMatch(dictionary, data, ring_buffer_mask, distance_cache, cur_ix, max_length, max_backward, gap, max_distance, out)
|
|
h.hb.FindLongestMatch(dictionary, data, ring_buffer_mask, distance_cache, cur_ix, max_length, max_backward, gap, max_distance, out)
|
|
}
|