otel-collector/exporter/datastoretracesexporter/factory.go

// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package datastoretracesexporter

import (
	"context"
	"fmt"
	"time"

	"github.com/hanzo-ds/go"
	"github.com/google/uuid"
	"github.com/jellydator/ttlcache/v3"
	"go.opentelemetry.io/collector/component"
	"go.opentelemetry.io/collector/config/configoptional"
	"go.opentelemetry.io/collector/config/configretry"
	"go.opentelemetry.io/collector/exporter"
	"go.opentelemetry.io/collector/exporter/exporterhelper"

	"github.com/hanzoai/otel-collector/exporter/datastoretracesexporter/internal/metadata"
)

func createDefaultConfig() component.Config {
	return &Config{
		TimeoutConfig:    exporterhelper.NewDefaultTimeoutConfig(),
		QueueBatchConfig: configoptional.Some(exporterhelper.NewDefaultQueueConfig()),
		BackOffConfig:    configretry.NewDefaultBackOffConfig(),
		AttributesLimits: AttributesLimits{
			FetchKeysInterval: 10 * time.Minute,
			MaxDistinctValues: 25000,
		},
	}
}

// NewFactory creates a factory for Logging exporter
func NewFactory() exporter.Factory {
	return exporter.NewFactory(
		metadata.Type,
		createDefaultConfig,
		exporter.WithTraces(createTracesExporter, metadata.TracesStability),
	)
}

func createTracesExporter(
	ctx context.Context,
	params exporter.Settings,
	cfg component.Config,
) (exporter.Traces, error) {

	c := cfg.(*Config)

	client, err := newDatastoreClient(ctx, c)
	if err != nil {
		return nil, fmt.Errorf("failed to create datastore client: %w", err)
	}

	id := uuid.New()

	// keys cache is used to avoid duplicate inserts for the same attribute key.
	keysCache := ttlcache.New(
		ttlcache.WithTTL[string, struct{}](240*time.Minute),
		ttlcache.WithCapacity[string, struct{}](50000),
		ttlcache.WithDisableTouchOnHit[string, struct{}](),
	)
	go keysCache.Start()

	// resource fingerprint cache is used to avoid duplicate inserts for the same resource fingerprint.
	// the ttl is set to the same as the bucket rounded value i.e 1800 seconds.
	// if a resource fingerprint is seen in the bucket already, skip inserting it again.
	rfCache := ttlcache.New(
		ttlcache.WithTTL[string, struct{}](distributedTracesResourceV2Seconds*time.Second),
		ttlcache.WithDisableTouchOnHit[string, struct{}](),
		ttlcache.WithCapacity[string, struct{}](100000),
	)
	go rfCache.Start()

	meter := params.MeterProvider.Meter(metadata.ScopeName)
	writerOpts := []WriterOption{
		WithDatastoreClient(client),
		WithLogger(params.Logger),
		WithMeter(meter),
		WithKeysCache(keysCache),
		WithRFCache(rfCache),
		WithAttributesLimits(c.AttributesLimits),
		WithExporterID(id),
	}

	exporterOpts := []TraceExporterOption{
		WithNewUsageCollector(id, client, params.Logger),
	}

	exporter, err := newExporter(c, params, writerOpts, exporterOpts)
	if err != nil {
		return nil, err
	}

	return exporterhelper.NewTraces(
		ctx,
		params,
		cfg,
		exporter.pushTraceData,
		exporterhelper.WithShutdown(exporter.Shutdown),
		exporterhelper.WithTimeout(c.TimeoutConfig),
		exporterhelper.WithQueue(c.QueueBatchConfig),
		exporterhelper.WithRetry(c.BackOffConfig))
}

func newDatastoreClient(ctx context.Context, cfg *Config) (datastore.Conn, error) {
	options, err := datastore.ParseDSN(cfg.Datasource)
	if err != nil {
		return nil, err
	}
	// setting maxIdleConnections = numConsumers + 1 to avoid `prepareBatch:datastore: acquire conn timeout` error
	// default to 1 extra idle connection; if queue config present, align with consumer count.
	maxIdleConnections := 1
	if qc := cfg.QueueBatchConfig.Get(); qc != nil {
		maxIdleConnections = qc.NumConsumers + 1
	}
	if options.MaxIdleConns < maxIdleConnections {
		options.MaxIdleConns = maxIdleConnections
		options.MaxOpenConns = maxIdleConnections + 5
	}
	db, err := datastore.Open(options)
	if err != nil {
		return nil, err
	}
	if err := db.Ping(ctx); err != nil {
		return nil, err
	}
	return db, nil
}