mirror of
https://github.com/hanzoai/traces.git
synced 2026-10-30 18:20:41 +00:00
fc98766c9a
* tempodb: update poller tests and mocks to support new work * include Blocklist interface for testing * tempodb: include quick poller * Include QuickBlocks on tempodb * Include notes about what is being implemented * Add initial ListBlocks() stubs and implement s3 * Fix poller commits * Lint fix * Fix method for unimplemented test * Add test coverage for previous compactions * Fix method for unimplemented test * Lint fix * Drop some debug printing * Drop unused variablea and update debug message * Relocate allocations * Invent fewer objects * Lint fix * Implement gcs ListBlocks() * Tidy up s3.go * Import fix * Adjust list_test.go for eyeball parse * Include notes after discoverying why * Wrap a couple errors for clarity * Modify error test for additional details * Modify gcs backend for single character delimiter * Add note about call change * Add note about call change and implement listv2 for s3 * Adjust uuidInBlocklist to use existing allocations * Add config and defaults for tenant polling concurrency * Implement concurrent tenant polling * Implement Find() on backend readers * Add Find to MockRawReader * Convert QuickBlocks over to use Find() * Implement Find() on local/cache backend readers * Drop ListBlocks in favor of Find * Replace Blocks() with QuickBlocks() and update usage everywhere * Fix local.Find for path handling * Fix backend raw tests for Find mocks * Reorder to reduce PR diff * Relocate super-handy block boudary method * Extend FindFunc to indicate Done * Parallelize the Find() calls within Blocks() * Fix race * Fix boundedwaitgroup init in test * Update backend implementations * Make note of implemenation details * Capture errors from tenant polling * Add feature detection on backend readers * Use backend.HasFeature() to determine if appropriate function * Fix error handling in poller * Pick lint * Extend log message to include tenant_concurrency * Add doc for HasFeature * Local backend supports no features * Pick lint * Ensure we wait for results * Avoid early return * Improve error handling to include all errors * Remove HasFeature() from RawReader interface after discussion * Drop start argument from Find signature * Include range calculation for sharded list calls * Enable concurrent list calls for s3.Find() * Include useful OwnsEverythingSharder * Remove concurrency handling from tempodb.Blocks() * Add docs for new config options * Fix GCS e2e object path * Clean up comments * Include a test to ensure retained blocks are forgotten * Begin reimplementing ListBlock() in RawReaders after discussion * Reimplement ListBlocks() on the backends * Fix ListBlocks() for gcs * Drop Find and supporting code * Drop unused method * Back out proto change, since `main` also has this issue * Update docs/sources/tempo/configuration/_index.md Co-authored-by: Mario <[email protected]> * Reimplement ListBlocks() for Azure * Lint * Rename config option for concurrent block list * Relocate block boundary code to own package * Relocate Blocklist interface to poller package * Rely on error count for early exit condition * Avoid polling additional tenants if we have already reached our error tollerance * Improve tests * Include initial benchmarks * Update docs/sources/tempo/configuration/_index.md Co-authored-by: Koenraad Verheyden <[email protected]> * Update docs/sources/tempo/configuration/_index.md Co-authored-by: Koenraad Verheyden <[email protected]> * Use *List directly, drop Blocklist interface * Use builtin clear() to avoid heap escape * Adhere to new error usage * Speed up benchmark setup * Trade memory for time in large data sets * Lock for race * Update CLI handling for Blocks() call to include compacted results * Add doc to mention the concurrency per-instance * Tidy up backends a bit * Implement concurrent ListBlocks() on gcs * Clean up * Include ListBlocksConcurrency config option for GCS * Test for compacted migration * Start tracing ListBlocks() * Start tracing the poller * Fix span attribute * Sort the block list for test * Use slice collection instead of channel collection for deadlock * Revert tenant poll concurrency * Pass context to the EnablePolling method for future use * Handle context on pollingLoop * Use attribtues not events for better traceql * Drop last tenant concurrency reference * Fix Do() revert to old code * Update for context * Update tracing attributes for poller * Back out proto update * Remove unused error * Remove inacurate docs after refactor * Clean up * Include additional span attributes * Drop unused * Drop unused * Address some feedback in the backend * Skip compacted blocks on trace summary * Include TODO about reducing the 404 on meta.json for compacted blocks * Include integration test for poller * Upgrade fake-gcs-server * Use JSONRead in gcs client for fake-gcs-server when Endpoint is set * Avoid early allocation of error for concurent operations * Include EndOffset in gcs backend * Include integration configs * Fix for global min/max * Include poller integration test in ci * Preallocate for review comments * Lock for all results and break early if error * Include additional assertion for poller_test * Adjust gcs and s3 locking * Adjust ListBlocks interface to take a tenant string instead of a keypath * Move OwnsEverythingSharder * Remove some comments * Ensure exclusive max unless global max * Restructure for clarity * Tidy up backends a bit * Fix test call count after avoiding BlockMeta for known compacted blocks * Update tests to include CompactedMeta call count * Include error message when block UUID is below shard minimum * Centralize backend.GlobalMaxBlockID --------- Co-authored-by: Mario <[email protected]> Co-authored-by: Koenraad Verheyden <[email protected]>
41 lines
1.0 KiB
Go
41 lines
1.0 KiB
Go
package blockboundary
|
|
|
|
import (
|
|
"encoding/binary"
|
|
"math"
|
|
)
|
|
|
|
// CreateBlockBoundaries splits the range of blockIDs into queryShards parts
|
|
func CreateBlockBoundaries(queryShards int) [][]byte {
|
|
if queryShards == 0 {
|
|
return nil
|
|
}
|
|
|
|
// create sharded queries
|
|
blockBoundaries := make([][]byte, queryShards+1)
|
|
for i := 0; i < queryShards+1; i++ {
|
|
blockBoundaries[i] = make([]byte, 16)
|
|
}
|
|
|
|
// bucketSz is the min size for the bucket
|
|
bucketSz := (math.MaxUint64 / uint64(queryShards))
|
|
// numLarger is the number of buckets that have to be bumped by 1
|
|
numLarger := (math.MaxUint64 % uint64(queryShards))
|
|
boundary := uint64(0)
|
|
for i := 0; i < queryShards; i++ {
|
|
binary.BigEndian.PutUint64(blockBoundaries[i][:8], boundary)
|
|
binary.BigEndian.PutUint64(blockBoundaries[i][8:], 0)
|
|
|
|
boundary += bucketSz
|
|
if numLarger != 0 {
|
|
numLarger--
|
|
boundary++
|
|
}
|
|
}
|
|
|
|
binary.BigEndian.PutUint64(blockBoundaries[queryShards][:8], math.MaxUint64)
|
|
binary.BigEndian.PutUint64(blockBoundaries[queryShards][8:], math.MaxUint64)
|
|
|
|
return blockBoundaries
|
|
}
|