mirror of
https://github.com/hanzoai/traces.git
synced 2026-10-30 18:20:41 +00:00
Zach LeslieandGitHub
a9f63d073b
* Update proto for v1 backend * Functioning blocklist with new package * Functioning blocklist with existing package * Generate proto to existing package * Add wrapper around google UUID for proto marshaling * Update blocklist for wrapper type * Test WriteTenantIndex * Implement json for uuid * Almost functional backend tests * Passing backend tests * Improve UUID coverage and utility * TempoDB updates for UUID and struct type changes * Update method style for uuid.New() * Update method style for uuid.MustParse() * Add uuid.From helper * Update modules for UUID and proto struct types * Update poller integration test for UUID * Update tempo-cli for UUID and proto struct type changes * Lint and clean up * Fix serverless * Fix cli * Fix cli * Add fixture tests for roundtrip assurance between versions * Clean up notes from proto definition * Update backend/test * Be more specific about unmarshal UUID * Include event for compacted block * Use better json tag for compacted meta * Implement UnmarshalJson on CompcatedBlockMeta for embedded struct * Test for CompactedMeta marshal/unmarshal * Drop benchmark index * Clean commented code * Extend mocks to capture multiple writes * Update backend/test fixtures * Write both proto and json to the backend for all three files in all backends * Fix lint * Back UUID with google.UUID to avoid struct * Relocate pkg/uuid to tempodb/backend * Re-plumb UUID through tempodb * Re-plumb UUID through cmd * Re-plumb UUID through modules * Re-plumb UUID through integration/ * Drop package spec from tempo.proto * Improve encoding marshal performance * Drop block proto and tenantindex json * Drop additional proto include * Implement zstd tenant index compression * Add note about use of json in the dedicatedcolumns * Fix ingester local block * Spell * Revert mock update * Drop json fallback from tenantindex read * Replace single-tenant test fixtures * Update fixture replacement test * Improve error handling and fix test * Extend test for tenantindex round-trip * Small fixes * Revert "Drop json fallback from tenantindex read" This reverts commit 834ff5b85d8a8a4d7eb560fbb67ba8f3a5a3959a. * Add back mocks to capture multiple backend writes * Add back proto & json writitng/reading for tenantindex * Change TotalObjects type from int32 -> int64 The original change was int -> int32, which on 64bit machines is a downgrade that I overlooked. Raising here to int64 to maintain the size. * Store write buffer at object path in mocks to improve readability in tests * Avoid allocation in interface validation * Update note to include intern * Pluralize metas and compacted_metas fields * Fallback to json only on ErrDoesNotExist * Lint * Drop debug * Revert "Pluralize metas and compacted_metas fields" This reverts commit 46a8c36e37ac33e1ffc104287d898930e02cdd97. * Import original tenantindex and test contents for backwards compatibility * Update changelog
95 lines
2.6 KiB
Go
95 lines
2.6 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
|
|
"github.com/dustin/go-humanize"
|
|
"github.com/google/uuid"
|
|
|
|
"github.com/grafana/tempo/tempodb/backend"
|
|
"github.com/grafana/tempo/tempodb/encoding"
|
|
)
|
|
|
|
type migrateTenantCmd struct {
|
|
SourceConfigFile string `type:"path" required:"" help:"Path to tempo config file for source"`
|
|
|
|
SourceTenantID string `arg:"" help:"source tenant-id"`
|
|
DestTenantID string `arg:"" help:"dest tenant-id"`
|
|
}
|
|
|
|
func (cmd *migrateTenantCmd) Run(opts *globalOptions) error {
|
|
ctx := context.Background()
|
|
|
|
readerSource, readerDest, writerDest, err := cmd.setupBackends(opts)
|
|
if err != nil {
|
|
return fmt.Errorf("setting up backends: %w", err)
|
|
}
|
|
defer func() {
|
|
readerSource.Shutdown()
|
|
readerDest.Shutdown()
|
|
}()
|
|
|
|
sourceTenantIndex, err := readerSource.TenantIndex(ctx, cmd.SourceTenantID)
|
|
if err != nil {
|
|
return fmt.Errorf("reading source tenant index: %w", err)
|
|
}
|
|
fmt.Printf("Blocks in source: %d, compacted: %d\n", len(sourceTenantIndex.Meta), len(sourceTenantIndex.CompactedMeta))
|
|
|
|
// TODO create dest directory if it doesn't exist yet?
|
|
|
|
blocksDest, _, err := readerDest.Blocks(ctx, cmd.DestTenantID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("Blocks in destination: %d\n", len(blocksDest))
|
|
|
|
var copiedBlocks, copiedSize uint64
|
|
|
|
blocks:
|
|
for _, sourceBlockMeta := range sourceTenantIndex.Meta {
|
|
// check for collisions
|
|
for _, uuidDest := range blocksDest {
|
|
if (uuid.UUID)(sourceBlockMeta.BlockID) == uuidDest {
|
|
fmt.Printf("UUID %s exists in source and destination, skipping block\n", sourceBlockMeta.BlockID)
|
|
continue blocks
|
|
}
|
|
}
|
|
|
|
// create a copy with destination tenant ID
|
|
destBlockMeta := *sourceBlockMeta
|
|
destBlockMeta.TenantID = cmd.DestTenantID
|
|
|
|
encoder, err := encoding.FromVersion(sourceBlockMeta.Version)
|
|
if err != nil {
|
|
return fmt.Errorf("creating encoder from version: %w", err)
|
|
}
|
|
|
|
err = encoder.MigrateBlock(ctx, sourceBlockMeta, &destBlockMeta, readerSource, writerDest)
|
|
if err != nil {
|
|
return fmt.Errorf("copying block: %w", err)
|
|
}
|
|
|
|
copiedBlocks++
|
|
copiedSize += sourceBlockMeta.Size_
|
|
}
|
|
|
|
fmt.Printf("Finished migrating data. Copied %d blocks, %s\n", copiedBlocks, humanize.Bytes(copiedSize))
|
|
return nil
|
|
}
|
|
|
|
func (cmd *migrateTenantCmd) setupBackends(optsDest *globalOptions) (readerSource, readerDest backend.Reader, writerDest backend.Writer, err error) {
|
|
emptyBackendOptions := backendOptions{}
|
|
|
|
optsSource := &globalOptions{
|
|
ConfigFile: cmd.SourceConfigFile,
|
|
}
|
|
readerSource, _, _, err = loadBackend(&emptyBackendOptions, optsSource)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
readerDest, writerDest, _, err = loadBackend(&emptyBackendOptions, optsDest)
|
|
return
|
|
}
|