Files
zeekay ed4424bf84 merge: upstream/main (sync, brand preserved)
# Conflicts:
#	.github/workflows/build-community.yaml
#	.github/workflows/build-enterprise.yaml
#	.github/workflows/gor-signoz-community.yaml
#	cmd/community/Dockerfile
#	cmd/community/Dockerfile.multi-arch
#	cmd/community/main.go
#	cmd/community/server.go
#	cmd/enterprise/Dockerfile
#	cmd/enterprise/Dockerfile.integration
#	cmd/enterprise/Dockerfile.with-web.integration
#	cmd/server/metastore.go
#	cmd/zap.go
#	conf/prometheus.yml
#	deploy/docker-swarm/generator/hotrod/docker-compose.yaml
#	deploy/docker-swarm/generator/infra/docker-compose.yaml
#	deploy/docker-swarm/generator/infra/otel-agent-config.yaml
#	deploy/docker-swarm/generator/infra/otel-metrics-config.yaml
#	deploy/docker/generator/hotrod/docker-compose.yaml
#	deploy/docker/generator/infra/docker-compose.yaml
#	deploy/docker/generator/infra/otel-collector-config.yaml
#	docs/contributing/go/integration.md
#	ee/anomaly/hourly.go
#	ee/anomaly/params.go
#	ee/anomaly/seasonal.go
#	ee/authn/callbackauthn/oidccallbackauthn/authn.go
#	ee/authz/openfgaauthz/provider.go
#	ee/authz/openfgaschema/base.fga
#	ee/authz/openfgaserver/server.go
#	ee/licensing/config.go
#	ee/licensing/httplicensing/provider.go
#	ee/modules/dashboard/impldashboard/module.go
#	ee/querier/handler.go
#	ee/query-service/anomaly/seasonal.go
#	ee/query-service/app/api/cloudIntegrations.go
#	ee/query-service/app/api/featureFlags.go
#	ee/query-service/app/api/license.go
#	ee/query-service/app/api/queryrange.go
#	ee/query-service/rules/anomaly.go
#	ee/query-service/rules/anomaly_test.go
#	ee/query-service/rules/manager.go
#	ee/query-service/rules/manager_test.go
#	ee/query-service/usage/manager.go
#	ee/sqlschema/postgressqlschema/provider.go
#	ee/sqlstore/postgressqlstore/dialect.go
#	ee/sqlstore/postgressqlstore/provider.go
#	ee/zeus/config.go
#	ee/zeus/httpzeus/provider.go
#	frontend/.eslintrc.cjs
#	frontend/e2e/test-plan/README.md
#	frontend/e2e/test-plan/validation-report.md
#	frontend/e2e/tests/settings/account-settings/account-settings-settings.spec.ts
#	frontend/e2e/tests/settings/api-keys/api-keys-settings.spec.ts
#	frontend/playwright.config.ts
#	frontend/public/Images/signoz-hero-image.webp
#	frontend/public/Logos/signoz-brand-logo.svg
#	frontend/scripts/generate-permissions-type.cjs
#	frontend/src/assets/Images/o11y-hero-image.webp
#	frontend/src/assets/Images/signoz-hero-image.webp
#	frontend/src/assets/Logos/hanzo-brand-logo.svg
#	frontend/src/assets/Logos/hanzo-icon.svg
#	frontend/src/assets/Logos/o11y-brand-logo.svg
#	frontend/src/assets/Logos/signoz-brand-logo.svg
#	frontend/src/assets/SigNoz-dark.svg
#	frontend/src/assets/SigNoz-white.svg
#	frontend/src/assets/signoz.svg
#	frontend/src/components/DetailsDrawer/DetailsDrawer.styles.scss
#	frontend/src/components/HostMetricsDetail/HostMetricTraces/HostMetricTraces.styles.scss
#	frontend/src/components/HostMetricsDetail/HostMetricTraces/constants.ts
#	frontend/src/components/HostMetricsDetail/HostMetricsDetail.styles.scss
#	frontend/src/components/HostMetricsDetail/HostMetricsDetails.tsx
#	frontend/src/components/HostMetricsDetail/HostMetricsLogs/NoLogsContainer.tsx
#	frontend/src/components/HostMetricsDetail/HostMetricsLogs/constants.ts
#	frontend/src/components/LogsDownloadOptionsMenu/LogsDownloadOptionsMenu.styles.scss
#	frontend/src/components/WelcomeLeftContainer/index.tsx
#	frontend/src/container/APIKeys/APIKeys.styles.scss
#	frontend/src/container/APIKeys/APIKeys.test.tsx
#	frontend/src/container/APIKeys/APIKeys.tsx
#	frontend/src/container/CloudIntegrationPage/Header/Header.tsx
#	frontend/src/container/CloudIntegrationPage/HeroSection/HeroSection.tsx
#	frontend/src/container/CloudIntegrationPage/HeroSection/components/AccountActions.style.scss
#	frontend/src/container/CloudIntegrationPage/HeroSection/components/AccountActions.tsx
#	frontend/src/container/CloudIntegrationPage/HeroSection/components/AccountSettingsModal.style.scss
#	frontend/src/container/CloudIntegrationPage/HeroSection/components/AccountSettingsModal.tsx
#	frontend/src/container/CloudIntegrationPage/HeroSection/components/CloudAccountSetupModal.style.scss
#	frontend/src/container/CloudIntegrationPage/HeroSection/components/RemoveIntegrationAccount.tsx
#	frontend/src/container/CloudIntegrationPage/ServicesSection/ConfigureServiceModal.tsx
#	frontend/src/container/CloudIntegrationPage/ServicesSection/ServicesTabs.style.scss
#	frontend/src/container/CloudIntegrationPage/ServicesSection/ServicesTabs.tsx
#	frontend/src/container/DashboardContainer/DashboardSettings/General/AddTags/AddTags.styles.scss
#	frontend/src/container/DashboardContainer/DashboardSettings/General/GeneralSettings.styles.scss
#	frontend/src/container/FormAlertRules/ChQuerySection/ChQuerySection.tsx
#	frontend/src/container/FormAlertRules/UserGuide/index.tsx
#	frontend/src/container/InfraMonitoringHosts/HostsEmptyOrIncorrectMetrics.tsx
#	frontend/src/container/InfraMonitoringHosts/InfraMonitoring.styles.scss
#	frontend/src/container/InfraMonitoringHosts/__tests__/HostsEmptyOrIncorrectMetrics.test.tsx
#	frontend/src/container/InfraMonitoringK8s/Clusters/ClusterDetails/ClusterDetails.tsx
#	frontend/src/container/InfraMonitoringK8s/Clusters/utils.tsx
#	frontend/src/container/InfraMonitoringK8s/DaemonSets/DaemonSetDetails/DaemonSetDetails.tsx
#	frontend/src/container/InfraMonitoringK8s/DaemonSets/utils.tsx
#	frontend/src/container/InfraMonitoringK8s/Deployments/DeploymentDetails/DeploymentDetails.tsx
#	frontend/src/container/InfraMonitoringK8s/Deployments/utils.tsx
#	frontend/src/container/InfraMonitoringK8s/EntityDetailsUtils/EntityEvents/entityEvents.styles.scss
#	frontend/src/container/InfraMonitoringK8s/EntityDetailsUtils/EntityTraces/entityTraces.styles.scss
#	frontend/src/container/InfraMonitoringK8s/InfraMonitoringK8s.styles.scss
#	frontend/src/container/InfraMonitoringK8s/Jobs/JobDetails/JobDetails.tsx
#	frontend/src/container/InfraMonitoringK8s/Jobs/utils.tsx
#	frontend/src/container/InfraMonitoringK8s/K8sEmptyOrIncorrectMetrics.tsx
#	frontend/src/container/InfraMonitoringK8s/K8sFiltersSidePanel/K8sFiltersSidePanel.styles.scss
#	frontend/src/container/InfraMonitoringK8s/Namespaces/NamespaceDetails/NamespaceDetails.styles.scss
#	frontend/src/container/InfraMonitoringK8s/Namespaces/NamespaceDetails/NamespaceDetails.tsx
#	frontend/src/container/InfraMonitoringK8s/Namespaces/utils.tsx
#	frontend/src/container/InfraMonitoringK8s/Nodes/NodeDetails/NodeDetails.tsx
#	frontend/src/container/InfraMonitoringK8s/Nodes/utils.tsx
#	frontend/src/container/InfraMonitoringK8s/Pods/PodDetails/PodDetails.tsx
#	frontend/src/container/InfraMonitoringK8s/StatefulSets/StatefulSetDetails/StatefulSetDetails.tsx
#	frontend/src/container/InfraMonitoringK8s/StatefulSets/utils.tsx
#	frontend/src/container/InfraMonitoringK8s/Volumes/VolumeDetails/VolumeDetails.tsx
#	frontend/src/container/InfraMonitoringK8s/Volumes/utils.tsx
#	frontend/src/container/ListAlertRules/AlertsEmptyState/AlertsEmptyState.styles.scss
#	frontend/src/container/ListAlertRules/ListAlert.tsx
#	frontend/src/container/LogsContextList/styles.ts
#	frontend/src/container/LogsExplorerList/ColumnView/ColumnView.tsx
#	frontend/src/container/LogsExplorerViews/QueryStatus.tsx
#	frontend/src/container/MetricsApplication/Tabs/Overview/ApDex/ApDexTraces.tsx
#	frontend/src/container/MetricsExplorer/Explorer/RelatedMetrics.tsx
#	frontend/src/container/MetricsExplorer/Explorer/useGetRelatedMetricsGraphs.ts
#	frontend/src/container/MetricsExplorer/Summary/MetricTypeSearch.tsx
#	frontend/src/container/NewWidget/LeftContainer/QuerySection/QueryBuilder/ClickHouse/index.tsx
#	frontend/src/container/NewWidget/LeftContainer/QuerySection/QueryBuilder/ClickHouse/query.tsx
#	frontend/src/container/NewWidget/LeftContainer/QuerySection/QueryBuilder/clickHouse/index.tsx
#	frontend/src/container/NewWidget/LeftContainer/QuerySection/QueryBuilder/clickHouse/query.tsx
#	frontend/src/container/NewWidget/LeftContainer/QuerySection/QueryBuilder/datastore/index.tsx
#	frontend/src/container/NewWidget/LeftContainer/QuerySection/QueryBuilder/datastore/query.tsx
#	frontend/src/container/OnboardingQuestionaire/OnboardingFooter/OnboardingFooter.tsx
#	frontend/src/container/OnboardingQuestionaire/OnboardingHeader/OnboardingHeader.styles.scss
#	frontend/src/container/OnboardingQuestionaire/OnboardingHeader/OnboardingHeader.tsx
#	frontend/src/container/OnboardingV2Container/onboarding-configs/onboarding-config-with-links.json
#	frontend/src/container/QueryBuilder/QueryBuilder.tsx
#	frontend/src/container/QueryBuilder/components/Query/Query.tsx
#	frontend/src/container/RolesSettings/__tests__/utils.test.ts
#	frontend/src/container/ServiceTable/SkipOnBoardModal/index.tsx
#	frontend/src/container/TraceDetail/Missingtrace.tsx
#	frontend/src/container/TraceDetail/SelectedSpanDetails/config.ts
#	frontend/src/lib/uPlotV2/components/Tooltip/Tooltip.styles.scss
#	frontend/src/mocks-server/__mockdata__/apiKeys.ts
#	frontend/src/pages/AlertDetails/AlertHeader/AlertStatus/AlertStatus.tsx
#	frontend/src/pages/Integrations/IntegrationDetailPage/IntegrationDetailPage.styles.scss
#	frontend/src/pages/Integrations/IntegrationDetailPage/IntegrationDetailPage.tsx
#	frontend/src/pages/Integrations/IntegrationsList.tsx
#	frontend/src/pages/Integrations/utils.ts
#	frontend/src/types/api/integrations/aws.ts
#	pkg/alertmanager/o11yalertmanager/handler.go
#	pkg/apiserver/o11yapiserver/alertmanager.go
#	pkg/apiserver/o11yapiserver/cloudintegration.go
#	pkg/apiserver/o11yapiserver/inframonitoring.go
#	pkg/apiserver/o11yapiserver/llmpricingrule.go
#	pkg/apiserver/o11yapiserver/rawdataexport.go
#	pkg/apiserver/o11yapiserver/registry.go
#	pkg/apiserver/o11yapiserver/ruler.go
#	pkg/apiserver/o11yapiserver/ruler_examples.go
#	pkg/apiserver/o11yapiserver/ruler_examples_test.go
#	pkg/apiserver/o11yapiserver/rulestatehistory.go
#	pkg/apiserver/o11yapiserver/spanmapper.go
#	pkg/apiserver/o11yapiserver/tracedetail.go
#	pkg/authn/callbackauthn/googlecallbackauthn/authn.go
#	pkg/authz/openfgaauthz/provider.go
#	pkg/authz/openfgaschema/base.fga
#	pkg/authz/openfgaserver/server.go
#	pkg/authz/openfgaserver/server_test.go
#	pkg/authz/openfgaserver/sqlstore.go
#	pkg/http/middleware/api_key.go
#	pkg/http/middleware/authn.go
#	pkg/instrumentation/metric.go
#	pkg/modules/user/impluser/module.go
#	pkg/prometheus/clickhouseprometheus/client.go
#	pkg/prometheus/clickhouseprometheus/client_query_test.go
#	pkg/prometheus/clickhouseprometheus/provider.go
#	pkg/prometheus/config.go
#	pkg/prometheus/engine.go
#	pkg/prometheus/label.go
#	pkg/prometheus/prometheus.go
#	pkg/prometheus/prometheustest/provider.go
#	pkg/prometheus/prometheustest/utils_test.go
#	pkg/querier/promql_query.go
#	pkg/query-service/app/cloudintegrations/Readme.md
#	pkg/query-service/app/cloudintegrations/accountsRepo.go
#	pkg/query-service/app/cloudintegrations/constants.go
#	pkg/query-service/app/cloudintegrations/controller.go
#	pkg/query-service/app/cloudintegrations/models.go
#	pkg/query-service/app/cloudintegrations/serviceconfig_db.go
#	pkg/query-service/app/cloudintegrations/services/definitions/aws/eks/assets/dashboards/containerinsights.json
#	pkg/query-service/app/cloudintegrations/services/definitions/aws/eks/assets/dashboards/containerinsights_dot.json
#	pkg/query-service/app/cloudintegrations/services/definitions/aws/eks/assets/dashboards/overview.json
#	pkg/query-service/app/cloudintegrations/services/definitions/aws/eks/assets/dashboards/overview_dot.json
#	pkg/query-service/app/cloudintegrations/services/models.go
#	pkg/query-service/app/cloudintegrations/services/services.go
#	pkg/query-service/app/cloudintegrations/services/services_test.go
#	pkg/query-service/app/integrations/builtin_integrations/clickhouse/assets/dashboards/overview_dot.json
#	pkg/query-service/app/integrations/builtin_integrations/datastore/assets/dashboards/overview_dot.json
#	pkg/query-service/app/metricsexplorer/parser.go
#	pkg/query-service/app/metricsexplorer/summary.go
#	pkg/query-service/app/querier/querier_test.go
#	pkg/query-service/app/querier/v2/querier_test.go
#	pkg/query-service/app/summary.go
#	pkg/query-service/model/metrics_explorer/summary.go
#	pkg/query-service/rules/base_rule_test.go
#	pkg/query-service/rules/manager_test.go
#	pkg/query-service/rules/prom_rule.go
#	pkg/query-service/rules/prom_rule_task.go
#	pkg/query-service/rules/prom_rule_test.go
#	pkg/query-service/rules/threshold_rule_test.go
#	pkg/querybuilder/resourcefilter/tables.go
#	pkg/queryparser/queryfilterextractor/promql.go
#	pkg/ruler/o11yruler/handler.go
#	pkg/sqlmigration/088_migrate_ii_dashboards/clickhouse/overview.json
#	pkg/telemetrymetadata/metadata_test.go
#	pkg/telemetrystore/datastoretelemetrystore/telemetry.go
#	pkg/types/authtypes/name.go
#	pkg/types/authtypes/object.go
#	pkg/types/authtypes/selector.go
#	pkg/types/authtypes/typeable.go
#	pkg/types/authtypes/typeable_anonymous.go
#	pkg/types/authtypes/typeable_metaresource.go
#	pkg/types/authtypes/typeable_metaresources.go
#	pkg/types/authtypes/typeable_organization.go
#	pkg/types/authtypes/typeable_role.go
#	pkg/types/authtypes/typeable_user.go
#	pkg/types/authtypes/uuid.go
#	pkg/types/ctxtypes/auth.go
#	pkg/types/factor_api_key.go
#	pkg/types/integrationtypes/integration.go
#	pkg/types/roletypes/store.go
#	pkg/types/ruletypes/maintenance.go
#	pkg/types/serviceaccounttypes/service_account_role.go
#	pkg/types/serviceaccounttypes/store.go
2026-06-02 10:54:25 -07:00

812 lines
23 KiB
Go

package querier
import (
"context"
"encoding/json"
"fmt"
"log/slog"
"slices"
"time"
"github.com/hanzoai/o11y/pkg/cache"
"github.com/hanzoai/o11y/pkg/errors"
"github.com/hanzoai/o11y/pkg/factory"
qbtypes "github.com/hanzoai/o11y/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/hanzoai/o11y/pkg/valuer"
)
// bucketCache implements the BucketCache interface.
type bucketCache struct {
cache cache.Cache
logger *slog.Logger
cacheTTL time.Duration
fluxInterval time.Duration
}
var _ BucketCache = (*bucketCache)(nil)
// NewBucketCache creates a new BucketCache implementation.
func NewBucketCache(settings factory.ProviderSettings, cache cache.Cache, cacheTTL time.Duration, fluxInterval time.Duration) BucketCache {
cacheSettings := factory.NewScopedProviderSettings(settings, "github.com/hanzoai/o11y/pkg/querier/bucket_cache")
return &bucketCache{
cache: cache,
logger: cacheSettings.Logger(),
cacheTTL: cacheTTL,
fluxInterval: fluxInterval,
}
}
// GetMissRanges returns cached data and missing time ranges.
func (bc *bucketCache) GetMissRanges(
ctx context.Context,
orgID valuer.UUID,
q qbtypes.Query,
step qbtypes.Step,
) (cached *qbtypes.Result, missing []*qbtypes.TimeRange) {
// Get query window
startMs, endMs := q.Window()
bc.logger.DebugContext(ctx, "getting miss ranges", slog.String("fingerprint", q.Fingerprint()), slog.Uint64("start", startMs), slog.Uint64("end", endMs))
// Generate cache key
cacheKey := bc.generateCacheKey(q)
bc.logger.DebugContext(ctx, "cache key", slog.String("cache_key", cacheKey))
// Try to get cached data
var data qbtypes.CachedData
err := bc.cache.Get(ctx, orgID, cacheKey, &data)
if err != nil {
if !errors.Ast(err, errors.TypeNotFound) {
bc.logger.ErrorContext(ctx, "error getting cached data", errors.Attr(err))
}
// No cached data, need to fetch entire range
missing = []*qbtypes.TimeRange{{From: startMs, To: endMs}}
return nil, missing
}
// Extract step interval if this is a builder query
stepMs := uint64(step.Milliseconds())
// Find missing ranges with step alignment
missing = bc.findMissingRangesWithStep(data.Buckets, startMs, endMs, stepMs)
bc.logger.DebugContext(ctx, "missing ranges", slog.Any("missing", missing), slog.Uint64("step", stepMs))
// If no cached data overlaps with requested range, return empty result
if len(data.Buckets) == 0 {
return nil, missing
}
// Extract relevant buckets and merge them
relevantBuckets := bc.filterRelevantBuckets(data.Buckets, startMs, endMs)
if len(relevantBuckets) == 0 {
return nil, missing
}
// Merge buckets into a single result
mergedResult := bc.mergeBuckets(ctx, relevantBuckets, data.Warnings)
// Filter the merged result to only include values within the requested time range
mergedResult = bc.filterResultToTimeRange(mergedResult, startMs, endMs)
return mergedResult, missing
}
// Put stores fresh query results in the cache.
func (bc *bucketCache) Put(ctx context.Context, orgID valuer.UUID, q qbtypes.Query, step qbtypes.Step, fresh *qbtypes.Result) {
// Get query window
startMs, endMs := q.Window()
// Calculate the flux boundary - data after this point should not be cached
currentMs := uint64(time.Now().UnixMilli())
fluxBoundary := currentMs - uint64(bc.fluxInterval.Milliseconds())
// If the entire range is within flux interval, skip caching
if startMs >= fluxBoundary {
bc.logger.DebugContext(ctx, "entire range within flux interval, skipping cache",
slog.Uint64("start", startMs),
slog.Uint64("end", endMs),
slog.Uint64("flux_boundary", fluxBoundary))
return
}
// Adjust endMs to not include data within flux interval
cachableEndMs := endMs
if endMs > fluxBoundary {
cachableEndMs = fluxBoundary
bc.logger.DebugContext(ctx, "adjusting end time to exclude flux interval",
slog.Uint64("original_end", endMs),
slog.Uint64("cachable_end", cachableEndMs))
}
// Generate cache key
cacheKey := bc.generateCacheKey(q)
// Get existing cached data
var existingData qbtypes.CachedData
if err := bc.cache.Get(ctx, orgID, cacheKey, &existingData); err != nil {
existingData = qbtypes.CachedData{}
}
// Trim the result to exclude data within flux interval
trimmedResult := bc.trimResultToFluxBoundary(fresh, cachableEndMs)
if trimmedResult == nil {
// Result type is not cacheable (raw or scalar)
return
}
// Adjust start and end times to only cache complete intervals
cachableStartMs := startMs
stepMs := uint64(step.Milliseconds())
// If we have a step interval, adjust boundaries to only cache complete intervals
if stepMs > 0 {
// If start is not aligned, round up to next step boundary (first complete interval)
if startMs%stepMs != 0 {
cachableStartMs = ((startMs / stepMs) + 1) * stepMs
}
// If end is not aligned, round down to previous step boundary (last complete interval)
if cachableEndMs%stepMs != 0 {
cachableEndMs = (cachableEndMs / stepMs) * stepMs
}
// If after adjustment we have no complete intervals, don't cache
if cachableStartMs >= cachableEndMs {
bc.logger.DebugContext(ctx, "no complete intervals to cache",
slog.Uint64("original_start", startMs),
slog.Uint64("original_end", endMs),
slog.Uint64("adjusted_start", cachableStartMs),
slog.Uint64("adjusted_end", cachableEndMs),
slog.Uint64("step", stepMs))
return
}
}
// Convert trimmed result to buckets with adjusted boundaries
freshBuckets := bc.resultToBuckets(ctx, trimmedResult, cachableStartMs, cachableEndMs)
// If no fresh buckets and no existing data, don't cache
if len(freshBuckets) == 0 && len(existingData.Buckets) == 0 {
return
}
// Merge with existing buckets
mergedBuckets := bc.mergeAndDeduplicateBuckets(existingData.Buckets, freshBuckets)
// Update warnings
allWarnings := append(existingData.Warnings, trimmedResult.Warnings...)
uniqueWarnings := bc.deduplicateWarnings(allWarnings)
// Create updated cached data
updatedData := qbtypes.CachedData{
Buckets: mergedBuckets,
Warnings: uniqueWarnings,
}
// Marshal and store in cache
if err := bc.cache.Set(ctx, orgID, cacheKey, &updatedData, bc.cacheTTL); err != nil {
bc.logger.ErrorContext(ctx, "error setting cached data", errors.Attr(err))
}
}
// generateCacheKey creates a unique cache key based on query fingerprint.
func (bc *bucketCache) generateCacheKey(q qbtypes.Query) string {
fingerprint := q.Fingerprint()
return fmt.Sprintf("v5:query:%s", fingerprint)
}
// findMissingRangesWithStep identifies time ranges not covered by cached buckets with step alignment.
func (bc *bucketCache) findMissingRangesWithStep(buckets []*qbtypes.CachedBucket, startMs, endMs uint64, stepMs uint64) []*qbtypes.TimeRange {
// When step is 0 or window is too small to be cached, use simple algorithm
if stepMs == 0 || (startMs+stepMs) > endMs {
return bc.findMissingRangesBasic(buckets, startMs, endMs)
}
// When no buckets exist, handle partial windows specially
if len(buckets) == 0 {
missing := make([]*qbtypes.TimeRange, 0, 3)
currentMs := startMs
// Check if start is not aligned - add partial window
if startMs%stepMs != 0 {
nextAggStart := startMs - (startMs % stepMs) + stepMs
missing = append(missing, &qbtypes.TimeRange{
From: startMs,
To: min(nextAggStart, endMs),
})
currentMs = nextAggStart
}
// Add the main range if needed
if currentMs < endMs {
missing = append(missing, &qbtypes.TimeRange{
From: currentMs,
To: endMs,
})
}
return missing
}
// Check if already sorted before sorting
needsSort := false
for i := 1; i < len(buckets); i++ {
if buckets[i].StartMs < buckets[i-1].StartMs {
needsSort = true
break
}
}
if needsSort {
slices.SortFunc(buckets, func(a, b *qbtypes.CachedBucket) int {
if a.StartMs < b.StartMs {
return -1
}
if a.StartMs > b.StartMs {
return 1
}
return 0
})
}
// Pre-allocate with reasonable capacity
missing := make([]*qbtypes.TimeRange, 0, len(buckets)+2)
currentMs := startMs
// Check if start is not aligned - add partial window
if startMs%stepMs != 0 {
nextAggStart := startMs - (startMs % stepMs) + stepMs
missing = append(missing, &qbtypes.TimeRange{
From: startMs,
To: min(nextAggStart, endMs),
})
currentMs = nextAggStart
}
for _, bucket := range buckets {
// Skip buckets that end before current position
if bucket.EndMs <= currentMs {
continue
}
// Stop processing if we've reached the end time
if bucket.StartMs >= endMs {
break
}
// Align bucket boundaries to step intervals
alignedBucketStart := bucket.StartMs
if bucket.StartMs%stepMs != 0 {
// Round up to next step boundary
alignedBucketStart = bucket.StartMs - (bucket.StartMs % stepMs) + stepMs
}
// Add gap before this bucket if needed
if currentMs < alignedBucketStart && currentMs < endMs {
missing = append(missing, &qbtypes.TimeRange{
From: currentMs,
To: min(alignedBucketStart, endMs),
})
}
// Update current position to the end of this bucket
// But ensure it's aligned to step boundary
bucketEnd := min(bucket.EndMs, endMs)
if bucketEnd%stepMs != 0 && bucketEnd < endMs {
// Round down to step boundary
bucketEnd = bucketEnd - (bucketEnd % stepMs)
}
currentMs = max(currentMs, bucketEnd)
}
// Add final gap if needed
if currentMs < endMs {
missing = append(missing, &qbtypes.TimeRange{
From: currentMs,
To: endMs,
})
}
// Don't merge ranges - keep partial windows separate for proper handling
return missing
}
// findMissingRangesBasic is the simple algorithm without step alignment.
func (bc *bucketCache) findMissingRangesBasic(buckets []*qbtypes.CachedBucket, startMs, endMs uint64) []*qbtypes.TimeRange {
// Check if already sorted before sorting
needsSort := false
for i := 1; i < len(buckets); i++ {
if buckets[i].StartMs < buckets[i-1].StartMs {
needsSort = true
break
}
}
if needsSort {
slices.SortFunc(buckets, func(a, b *qbtypes.CachedBucket) int {
if a.StartMs < b.StartMs {
return -1
}
if a.StartMs > b.StartMs {
return 1
}
return 0
})
}
// Pre-allocate with reasonable capacity
missing := make([]*qbtypes.TimeRange, 0, len(buckets)+1)
currentMs := startMs
for _, bucket := range buckets {
// Skip buckets that end before start time
if bucket.EndMs <= startMs {
continue
}
// Stop processing if we've reached the end time
if bucket.StartMs >= endMs {
break
}
// Add gap before this bucket if needed
if currentMs < bucket.StartMs {
missing = append(missing, &qbtypes.TimeRange{
From: currentMs,
To: min(bucket.StartMs, endMs),
})
}
// Update current position, but don't go past the end time
currentMs = max(currentMs, min(bucket.EndMs, endMs))
}
// Add final gap if needed
if currentMs < endMs {
// Check if we need to limit due to flux interval
currentTime := uint64(time.Now().UnixMilli())
fluxBoundary := currentTime - uint64(bc.fluxInterval.Milliseconds())
// If the missing range extends beyond flux boundary, limit it
if currentMs < fluxBoundary {
// Add range up to flux boundary
missing = append(missing, &qbtypes.TimeRange{
From: currentMs,
To: min(endMs, fluxBoundary),
})
// If endMs is beyond flux boundary, add that as another missing range
if endMs > fluxBoundary {
missing = append(missing, &qbtypes.TimeRange{
From: fluxBoundary,
To: endMs,
})
}
} else {
// Entire missing range is within flux interval
missing = append(missing, &qbtypes.TimeRange{
From: currentMs,
To: endMs,
})
}
}
// Don't merge ranges - keep partial windows separate for proper handling
return missing
}
// filterRelevantBuckets returns buckets that overlap with the requested time range.
func (bc *bucketCache) filterRelevantBuckets(buckets []*qbtypes.CachedBucket, startMs, endMs uint64) []*qbtypes.CachedBucket {
// Pre-allocate with estimated capacity
relevant := make([]*qbtypes.CachedBucket, 0, len(buckets))
for _, bucket := range buckets {
// Check if bucket overlaps with requested range
if bucket.EndMs > startMs && bucket.StartMs < endMs {
relevant = append(relevant, bucket)
}
}
// Sort by start time
slices.SortFunc(relevant, func(a, b *qbtypes.CachedBucket) int {
if a.StartMs < b.StartMs {
return -1
}
if a.StartMs > b.StartMs {
return 1
}
return 0
})
return relevant
}
// mergeBuckets combines multiple cached buckets into a single result.
func (bc *bucketCache) mergeBuckets(ctx context.Context, buckets []*qbtypes.CachedBucket, warnings []string) *qbtypes.Result {
if len(buckets) == 0 {
return &qbtypes.Result{}
}
// All buckets should have the same type
resultType := buckets[0].Type
// Aggregate stats
var totalStats qbtypes.ExecStats
for _, bucket := range buckets {
totalStats.RowsScanned += bucket.Stats.RowsScanned
totalStats.BytesScanned += bucket.Stats.BytesScanned
totalStats.DurationMS += bucket.Stats.DurationMS
}
// Merge values based on type
var mergedValue any
switch resultType {
case qbtypes.RequestTypeTimeSeries:
mergedValue = bc.mergeTimeSeriesValues(ctx, buckets)
// Raw and Scalar types are not cached, so no merge needed
}
return &qbtypes.Result{
Type: resultType,
Value: mergedValue,
Stats: totalStats,
Warnings: warnings,
}
}
// mergeTimeSeriesValues merges time series data from multiple buckets.
func (bc *bucketCache) mergeTimeSeriesValues(ctx context.Context, buckets []*qbtypes.CachedBucket) *qbtypes.TimeSeriesData {
// Estimate capacity based on bucket count
estimatedSeries := len(buckets) * 10
// Flat map with composite key for better performance
type seriesKey struct {
aggIndex int
key string
}
seriesMap := make(map[seriesKey]*qbtypes.TimeSeries, estimatedSeries)
for _, bucket := range buckets {
var tsData *qbtypes.TimeSeriesData
if err := json.Unmarshal(bucket.Value, &tsData); err != nil {
bc.logger.ErrorContext(ctx, "failed to unmarshal time series data", errors.Attr(err))
continue
}
for _, aggBucket := range tsData.Aggregations {
for _, series := range aggBucket.Series {
// Create series key from labels
key := seriesKey{
aggIndex: aggBucket.Index,
key: qbtypes.GetUniqueSeriesKey(series.Labels),
}
if existingSeries, ok := seriesMap[key]; ok {
// Merge values, avoiding duplicate timestamps
timestampMap := make(map[int64]bool)
for _, v := range existingSeries.Values {
timestampMap[v.Timestamp] = true
}
// Pre-allocate capacity for merged values
newCap := len(existingSeries.Values) + len(series.Values)
if cap(existingSeries.Values) < newCap {
newValues := make([]*qbtypes.TimeSeriesValue, len(existingSeries.Values), newCap)
copy(newValues, existingSeries.Values)
existingSeries.Values = newValues
}
// Only add values with new timestamps
for _, v := range series.Values {
if !timestampMap[v.Timestamp] {
existingSeries.Values = append(existingSeries.Values, v)
}
}
} else {
// New series
seriesMap[key] = series
}
}
}
}
// Group series by aggregation index
aggMap := make(map[int][]*qbtypes.TimeSeries)
for key, series := range seriesMap {
aggMap[key.aggIndex] = append(aggMap[key.aggIndex], series)
}
// Convert map back to slice
result := &qbtypes.TimeSeriesData{
Aggregations: make([]*qbtypes.AggregationBucket, 0, len(aggMap)),
}
for index, seriesList := range aggMap {
// Sort values by timestamp for each series
for _, s := range seriesList {
// Check if already sorted before sorting
needsSort := false
for i := 1; i < len(s.Values); i++ {
if s.Values[i].Timestamp < s.Values[i-1].Timestamp {
needsSort = true
break
}
}
if needsSort {
slices.SortFunc(s.Values, func(a, b *qbtypes.TimeSeriesValue) int {
if a.Timestamp < b.Timestamp {
return -1
}
if a.Timestamp > b.Timestamp {
return 1
}
return 0
})
}
}
result.Aggregations = append(result.Aggregations, &qbtypes.AggregationBucket{
Index: index,
Series: seriesList,
})
}
return result
}
// isEmptyResult checks if a result is truly empty (no data exists) vs filtered empty (data was filtered out).
func (bc *bucketCache) isEmptyResult(result *qbtypes.Result) (isEmpty bool, isFiltered bool) {
if result.Value == nil {
return true, false
}
switch result.Type {
case qbtypes.RequestTypeTimeSeries:
if tsData, ok := result.Value.(*qbtypes.TimeSeriesData); ok {
// No aggregations at all means truly empty
if len(tsData.Aggregations) == 0 {
return true, false
}
// Check if we have aggregations but no series (filtered out)
totalSeries := 0
for _, agg := range tsData.Aggregations {
totalSeries += len(agg.Series)
}
if totalSeries == 0 {
// We have aggregations but no series - data was filtered out
return true, true
}
// Check if all series have no values
hasValues := false
for _, agg := range tsData.Aggregations {
for _, series := range agg.Series {
if len(series.Values) > 0 {
hasValues = true
break
}
}
if hasValues {
break
}
}
return !hasValues, !hasValues && totalSeries > 0
}
case qbtypes.RequestTypeRaw, qbtypes.RequestTypeScalar, qbtypes.RequestTypeTrace:
// Raw and scalar data are not cached
return true, false
}
return true, false
}
// resultToBuckets converts a query result into time-based buckets.
func (bc *bucketCache) resultToBuckets(ctx context.Context, result *qbtypes.Result, startMs, endMs uint64) []*qbtypes.CachedBucket {
// Check if result is empty
isEmpty, isFiltered := bc.isEmptyResult(result)
// Don't cache if result is empty but not filtered
// Empty filtered results should be cached to avoid re-querying
if isEmpty && !isFiltered {
bc.logger.DebugContext(ctx, "skipping cache for empty non-filtered result")
return nil
}
// For now, create a single bucket for the entire range
// In the future, we could split large ranges into smaller buckets
valueBytes, err := json.Marshal(result.Value)
if err != nil {
bc.logger.ErrorContext(ctx, "failed to marshal result value", errors.Attr(err))
return nil
}
// Always create a bucket, even for empty filtered results
// This ensures we don't re-query for data that doesn't exist
return []*qbtypes.CachedBucket{
{
StartMs: startMs,
EndMs: endMs,
Type: result.Type,
Value: valueBytes,
Stats: result.Stats,
},
}
}
// mergeAndDeduplicateBuckets combines and deduplicates bucket lists.
func (bc *bucketCache) mergeAndDeduplicateBuckets(existing, fresh []*qbtypes.CachedBucket) []*qbtypes.CachedBucket {
// Create a map to deduplicate by time range
bucketMap := make(map[string]*qbtypes.CachedBucket)
// Add existing buckets
for _, bucket := range existing {
key := fmt.Sprintf("%d-%d", bucket.StartMs, bucket.EndMs)
bucketMap[key] = bucket
}
// Add/update with fresh buckets
for _, bucket := range fresh {
key := fmt.Sprintf("%d-%d", bucket.StartMs, bucket.EndMs)
bucketMap[key] = bucket
}
// Convert back to slice with pre-allocated capacity
result := make([]*qbtypes.CachedBucket, 0, len(bucketMap))
for _, bucket := range bucketMap {
result = append(result, bucket)
}
// Sort by start time
slices.SortFunc(result, func(a, b *qbtypes.CachedBucket) int {
if a.StartMs < b.StartMs {
return -1
}
if a.StartMs > b.StartMs {
return 1
}
return 0
})
return result
}
// deduplicateWarnings removes duplicate warnings.
func (bc *bucketCache) deduplicateWarnings(warnings []string) []string {
return dedupeWarnings(warnings)
}
// trimResultToFluxBoundary trims the result to exclude data points beyond the flux boundary.
func (bc *bucketCache) trimResultToFluxBoundary(result *qbtypes.Result, fluxBoundary uint64) *qbtypes.Result {
trimmedResult := &qbtypes.Result{
Type: result.Type,
Stats: result.Stats,
Warnings: result.Warnings,
}
switch result.Type {
case qbtypes.RequestTypeTimeSeries:
// Trim time series data
if tsData, ok := result.Value.(*qbtypes.TimeSeriesData); ok && tsData != nil {
trimmedData := &qbtypes.TimeSeriesData{}
for _, aggBucket := range tsData.Aggregations {
trimmedBucket := &qbtypes.AggregationBucket{
Index: aggBucket.Index,
}
for _, series := range aggBucket.Series {
trimmedSeries := &qbtypes.TimeSeries{
Labels: series.Labels,
}
// Filter values to exclude those beyond flux boundary and partial values
for _, value := range series.Values {
// Skip partial values - they cannot be cached
if value.Partial {
continue
}
if uint64(value.Timestamp) <= fluxBoundary {
trimmedSeries.Values = append(trimmedSeries.Values, value)
}
}
// Always add the series to preserve filtered empty results
trimmedBucket.Series = append(trimmedBucket.Series, trimmedSeries)
}
// Always add the bucket to preserve aggregation structure
trimmedData.Aggregations = append(trimmedData.Aggregations, trimmedBucket)
}
// Always set the value to preserve empty filtered results
trimmedResult.Value = trimmedData
}
case qbtypes.RequestTypeRaw, qbtypes.RequestTypeScalar, qbtypes.RequestTypeTrace:
// Don't cache raw or scalar data
return nil
}
return trimmedResult
}
func min(a, b uint64) uint64 {
if a < b {
return a
}
return b
}
func max(a, b uint64) uint64 {
if a > b {
return a
}
return b
}
// filterResultToTimeRange filters the result to only include values within the requested time range.
func (bc *bucketCache) filterResultToTimeRange(result *qbtypes.Result, startMs, endMs uint64) *qbtypes.Result {
if result == nil || result.Value == nil {
return result
}
switch result.Type {
case qbtypes.RequestTypeTimeSeries:
if tsData, ok := result.Value.(*qbtypes.TimeSeriesData); ok {
filteredData := &qbtypes.TimeSeriesData{
Aggregations: make([]*qbtypes.AggregationBucket, 0, len(tsData.Aggregations)),
}
for _, aggBucket := range tsData.Aggregations {
filteredBucket := &qbtypes.AggregationBucket{
Index: aggBucket.Index,
Alias: aggBucket.Alias,
Meta: aggBucket.Meta,
Series: make([]*qbtypes.TimeSeries, 0, len(aggBucket.Series)),
}
for _, series := range aggBucket.Series {
filteredSeries := &qbtypes.TimeSeries{
Labels: series.Labels,
Values: make([]*qbtypes.TimeSeriesValue, 0, len(series.Values)),
}
// Filter values to only include those within the requested time range
for _, value := range series.Values {
timestampMs := uint64(value.Timestamp)
if timestampMs >= startMs && timestampMs < endMs {
filteredSeries.Values = append(filteredSeries.Values, value)
}
}
// Always add series to preserve structure (even if empty)
filteredBucket.Series = append(filteredBucket.Series, filteredSeries)
}
// Only add bucket if it has series
if len(filteredBucket.Series) > 0 {
filteredData.Aggregations = append(filteredData.Aggregations, filteredBucket)
}
}
// Create a new result with the filtered data
return &qbtypes.Result{
Type: result.Type,
Value: filteredData,
Stats: result.Stats,
Warnings: result.Warnings,
}
}
}
// For non-time series data, return as is
return result
}