deploy/cmd/cd-repo-server/commands/cd_repo_server.go

package commands

import (
	stderrors "errors"
	"fmt"
	"math"
	"net"
	"net/http"
	"os"
	"os/signal"
	"runtime/debug"
	"sync"
	"syscall"
	"time"

	"github.com/hanzoai/cd/util/vendored/stats"
	"github.com/redis/go-redis/v9"
	log "github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
	"google.golang.org/grpc/health/grpc_health_v1"
	"k8s.io/apimachinery/pkg/api/resource"

	cmdutil "github.com/hanzoai/cd/cmd/util"
	reposervercache "github.com/hanzoai/cd/reposerver/cache"
	cacheutil "github.com/hanzoai/cd/util/cache"
	utilio "github.com/hanzoai/cd/util/io"

	"github.com/hanzoai/cd/common"
	"github.com/hanzoai/cd/reposerver"
	"github.com/hanzoai/cd/reposerver/apiclient"
	"github.com/hanzoai/cd/reposerver/metrics"
	"github.com/hanzoai/cd/reposerver/repository"
	"github.com/hanzoai/cd/util/askpass"
	"github.com/hanzoai/cd/util/cli"
	"github.com/hanzoai/cd/util/env"
	"github.com/hanzoai/cd/util/errors"
	"github.com/hanzoai/cd/util/healthz"
	"github.com/hanzoai/cd/util/profile"
	"github.com/hanzoai/cd/util/sourceintegrity"
	"github.com/hanzoai/cd/util/tls"
	traceutil "github.com/hanzoai/cd/util/trace"

	ctls "crypto/tls"
)

var (
	gnuPGSourcePath                              = env.StringFromEnv(common.EnvGPGDataPath, "/app/config/gpg/source")
	pauseGenerationAfterFailedGenerationAttempts = env.ParseNumFromEnv(common.EnvPauseGenerationAfterFailedAttempts, 3, 0, math.MaxInt32)
	pauseGenerationOnFailureForMinutes           = env.ParseNumFromEnv(common.EnvPauseGenerationMinutes, 60, 0, math.MaxInt32)
	pauseGenerationOnFailureForRequests          = env.ParseNumFromEnv(common.EnvPauseGenerationRequests, 0, 0, math.MaxInt32)
	gitSubmoduleEnabled                          = env.ParseBoolFromEnv(common.EnvGitSubmoduleEnabled, true)
	helmUserAgent                                = env.StringFromEnv(common.EnvHelmUserAgent, "")
)

func NewCommand() *cobra.Command {
	var (
		parallelismLimit                   int64
		listenPort                         int
		listenHost                         string
		metricsPort                        int
		metricsHost                        string
		otlpAddress                        string
		otlpInsecure                       bool
		otlpHeaders                        map[string]string
		otlpAttrs                          []string
		otlpSampleRatio                    float64
		cacheSrc                           func() (*reposervercache.Cache, error)
		tlsConfigCustomizer                tls.ConfigCustomizer
		tlsConfigCustomizerSrc             func() (tls.ConfigCustomizer, error)
		redisClient                        *redis.Client
		maxCombinedDirectoryManifestsSize  string
		cmpTarExcludedGlobs                []string
		allowOutOfBoundsSymlinks           bool
		streamedManifestMaxTarSize         string
		streamedManifestMaxExtractedSize   string
		helmManifestMaxExtractedSize       string
		helmRegistryMaxIndexSize           string
		ociManifestMaxExtractedSize        string
		disableOCIManifestMaxExtractedSize bool
		disableManifestMaxExtractedSize    bool
		includeHiddenDirectories           bool
		cmpUseManifestGeneratePaths        bool
		ociMediaTypes                      []string
		enableBuiltinGitConfig             bool
		clientCAPath                       string
		disableTLS                         bool
	)
	command := cobra.Command{
		Use:               common.CommandRepoServer,
		Short:             "Run Hanzo CD Repository Server",
		Long:              "Hanzo CD Repository Server is an internal service which maintains a local cache of the Git repository holding the application manifests, and is responsible for generating and returning the Kubernetes manifests.  This command runs Repository Server in the foreground.  It can be configured by following options.",
		DisableAutoGenTag: true,
		RunE: func(c *cobra.Command, _ []string) error {
			ctx := c.Context()

			vers := common.GetVersion()
			vers.LogStartupInfo(
				"Hanzo CD Repository Server",
				map[string]any{
					"port": listenPort,
				},
			)

			cli.SetLogFormat(cmdutil.LogFormat)
			cli.SetLogLevel(cmdutil.LogLevel)

			// Recover from panic and log the error using the configured logger instead of the default.
			defer func() {
				if r := recover(); r != nil {
					log.WithField("trace", string(debug.Stack())).Fatal("Recovered from panic: ", r)
				}
			}()

			if !disableTLS {
				var err error
				tlsConfigCustomizer, err = tlsConfigCustomizerSrc()
				errors.CheckError(err)
			}

			cache, err := cacheSrc()
			errors.CheckError(err)
			repoCacheExpiration := cache.GetRepoCacheExpiration()
			maxCombinedDirectoryManifestsQuantity, err := resource.ParseQuantity(maxCombinedDirectoryManifestsSize)
			errors.CheckError(err)

			streamedManifestMaxTarSizeQuantity, err := resource.ParseQuantity(streamedManifestMaxTarSize)
			errors.CheckError(err)

			streamedManifestMaxExtractedSizeQuantity, err := resource.ParseQuantity(streamedManifestMaxExtractedSize)
			errors.CheckError(err)

			helmManifestMaxExtractedSizeQuantity, err := resource.ParseQuantity(helmManifestMaxExtractedSize)
			errors.CheckError(err)

			ociManifestMaxExtractedSizeQuantity, err := resource.ParseQuantity(ociManifestMaxExtractedSize)
			errors.CheckError(err)

			helmRegistryMaxIndexSizeQuantity, err := resource.ParseQuantity(helmRegistryMaxIndexSize)
			errors.CheckError(err)

			askPassServer := askpass.NewServer(askpass.SocketPath)
			metricsServer := metrics.NewMetricsServer()
			cacheutil.CollectMetrics(redisClient, metricsServer, nil)
			if disableTLS && clientCAPath != "" {
				return stderrors.New("--client-ca-path cannot be used when --disable-tls is enabled")
			}

			server, err := reposerver.NewServer(metricsServer, cache, tlsConfigCustomizer, repository.RepoServerInitConstants{
				ParallelismLimit: parallelismLimit,
				PauseGenerationAfterFailedGenerationAttempts: pauseGenerationAfterFailedGenerationAttempts,
				PauseGenerationOnFailureForMinutes:           pauseGenerationOnFailureForMinutes,
				PauseGenerationOnFailureForRequests:          pauseGenerationOnFailureForRequests,
				SubmoduleEnabled:                             gitSubmoduleEnabled,
				MaxCombinedDirectoryManifestsSize:            maxCombinedDirectoryManifestsQuantity,
				CMPTarExcludedGlobs:                          cmpTarExcludedGlobs,
				AllowOutOfBoundsSymlinks:                     allowOutOfBoundsSymlinks,
				StreamedManifestMaxExtractedSize:             streamedManifestMaxExtractedSizeQuantity.ToDec().Value(),
				StreamedManifestMaxTarSize:                   streamedManifestMaxTarSizeQuantity.ToDec().Value(),
				HelmManifestMaxExtractedSize:                 helmManifestMaxExtractedSizeQuantity.ToDec().Value(),
				HelmRegistryMaxIndexSize:                     helmRegistryMaxIndexSizeQuantity.ToDec().Value(),
				OCIManifestMaxExtractedSize:                  ociManifestMaxExtractedSizeQuantity.ToDec().Value(),
				DisableOCIManifestMaxExtractedSize:           disableOCIManifestMaxExtractedSize,
				IncludeHiddenDirectories:                     includeHiddenDirectories,
				CMPUseManifestGeneratePaths:                  cmpUseManifestGeneratePaths,
				OCIMediaTypes:                                ociMediaTypes,
				EnableBuiltinGitConfig:                       enableBuiltinGitConfig,
				HelmUserAgent:                                helmUserAgent,
				HelmChartCacheExpiration:                     repoCacheExpiration,
			}, askPassServer, clientCAPath, disableTLS)
			errors.CheckError(err)

			if otlpAddress != "" {
				var closer func()
				var err error
				closer, err = traceutil.InitTracer(ctx, "cd-repo-server", otlpAddress, otlpInsecure, otlpHeaders, otlpAttrs, otlpSampleRatio)
				if err != nil {
					log.Fatalf("failed to initialize tracing: %v", err)
				}
				defer closer()
			}

			grpc := server.CreateGRPC()
			lc := &net.ListenConfig{}
			listener, err := lc.Listen(ctx, "tcp", fmt.Sprintf("%s:%d", listenHost, listenPort))
			errors.CheckError(err)

			healthCheckTLSConfig := buildHealthCheckTLSConfig(server.GetHealthCheckClientCert(), disableTLS)

			mux := http.NewServeMux()
			healthz.ServeHealthCheck(mux, func(r *http.Request) error {
				if val, ok := r.URL.Query()["full"]; ok && len(val) > 0 && val[0] == "true" {
					// connect to itself to make sure the repo server is able to serve the connection
					// used by liveness probe to auto-restart repo server
					// see https://github.com/argoproj/argo-cd/issues/5110 for more information
					conn, err := apiclient.NewConnection(fmt.Sprintf("localhost:%d", listenPort), 60, healthCheckTLSConfig)
					if err != nil {
						return err
					}
					defer utilio.Close(conn)
					client := grpc_health_v1.NewHealthClient(conn)
					res, err := client.Check(r.Context(), &grpc_health_v1.HealthCheckRequest{})
					if err != nil {
						return err
					}
					if res.Status != grpc_health_v1.HealthCheckResponse_SERVING {
						return fmt.Errorf("grpc health check status is '%v'", res.Status)
					}
					return nil
				}
				return nil
			})
			mux.Handle("/metrics", metricsServer.GetHandler())
			profile.RegisterProfiler(mux)
			go func() { errors.CheckError(http.ListenAndServe(fmt.Sprintf("%s:%d", metricsHost, metricsPort), mux)) }()
			go func() { errors.CheckError(askPassServer.Run()) }()

			if sourceintegrity.IsGPGEnabled() {
				log.Infof("Initializing GnuPG keyring at %s", common.GetGnuPGHomePath())
				err = sourceintegrity.InitializeGnuPG()
				errors.CheckError(err)

				log.Infof("Populating GnuPG keyring with keys from %s", gnuPGSourcePath)
				added, removed, err := sourceintegrity.SyncKeyRingFromDirectory(gnuPGSourcePath)
				errors.CheckError(err)
				log.Infof("Loaded %d (and removed %d) keys from keyring", len(added), len(removed))

				go func() { errors.CheckError(reposerver.StartGPGWatcher(gnuPGSourcePath)) }()
			}

			log.Infof("cd-repo-server is listening on %s", listener.Addr())
			stats.RegisterStackDumper()
			stats.StartStatsTicker(10 * time.Minute)
			stats.RegisterHeapDumper("memprofile")

			// Graceful shutdown code adapted from https://gist.github.com/embano1/e0bf49d24f1cdd07cffad93097c04f0a
			sigCh := make(chan os.Signal, 1)
			signal.Notify(sigCh, os.Interrupt, syscall.SIGTERM)
			wg := sync.WaitGroup{}
			wg.Go(func() {
				s := <-sigCh
				log.Printf("got signal %v, attempting graceful shutdown", s)
				grpc.GracefulStop()
			})

			log.Println("starting grpc server")
			err = grpc.Serve(listener)
			if err != nil {
				log.Fatalf("could not serve: %v", err)
			}
			wg.Wait()
			log.Println("clean shutdown")

			return nil
		},
	}
	command.Flags().StringVar(&cmdutil.LogFormat, "logformat", env.StringFromEnv("CD_REPO_SERVER_LOGFORMAT", "json"), "Set the logging format. One of: json|text")
	command.Flags().StringVar(&cmdutil.LogLevel, "loglevel", env.StringFromEnv("CD_REPO_SERVER_LOGLEVEL", "info"), "Set the logging level. One of: debug|info|warn|error")
	command.Flags().Int64Var(&parallelismLimit, "parallelismlimit", int64(env.ParseNumFromEnv("CD_REPO_SERVER_PARALLELISM_LIMIT", 0, 0, math.MaxInt32)), "Limit on number of concurrent manifests generate requests. Any value less the 1 means no limit.")
	command.Flags().StringVar(&listenHost, "address", env.StringFromEnv("CD_REPO_SERVER_LISTEN_ADDRESS", common.DefaultAddressRepoServer), "Listen on given address for incoming connections")
	command.Flags().IntVar(&listenPort, "port", common.DefaultPortRepoServer, "Listen on given port for incoming connections")
	command.Flags().StringVar(&metricsHost, "metrics-address", env.StringFromEnv("CD_REPO_SERVER_METRICS_LISTEN_ADDRESS", common.DefaultAddressRepoServerMetrics), "Listen on given address for metrics")
	command.Flags().IntVar(&metricsPort, "metrics-port", common.DefaultPortRepoServerMetrics, "Start metrics server on given port")
	command.Flags().StringVar(&otlpAddress, "otlp-address", env.StringFromEnv("CD_REPO_SERVER_OTLP_ADDRESS", ""), "OpenTelemetry collector address to send traces to")
	command.Flags().BoolVar(&otlpInsecure, "otlp-insecure", env.ParseBoolFromEnv("CD_REPO_SERVER_OTLP_INSECURE", true), "OpenTelemetry collector insecure mode")
	command.Flags().StringToStringVar(&otlpHeaders, "otlp-headers", env.ParseStringToStringFromEnv("CD_REPO_SERVER_OTLP_HEADERS", map[string]string{}, ","), "List of OpenTelemetry collector extra headers sent with traces, headers are comma-separated key-value pairs(e.g. key1=value1,key2=value2)")
	command.Flags().StringSliceVar(&otlpAttrs, "otlp-attrs", env.StringsFromEnv("CD_REPO_SERVER_OTLP_ATTRS", []string{}, ","), "List of OpenTelemetry collector extra attrs when send traces, each attribute is separated by a colon(e.g. key:value)")
	cli.BoundedFloat64Var(command.Flags(), &otlpSampleRatio, "otlp-sample-ratio", env.ParseFloat64FromEnv("CD_REPO_SERVER_OTLP_SAMPLE_RATIO", 1.0, 0.0, 1.0), 0.0, 1.0, "Fraction of traces to sample, from 0.0 (none) to 1.0 (all). Parent-based, so downstream services honor the upstream sampling decision")
	command.Flags().StringVar(&maxCombinedDirectoryManifestsSize, "max-combined-directory-manifests-size", env.StringFromEnv("CD_REPO_SERVER_MAX_COMBINED_DIRECTORY_MANIFESTS_SIZE", "10M"), "Max combined size of manifest files in a directory-type Application")
	command.Flags().StringArrayVar(&cmpTarExcludedGlobs, "plugin-tar-exclude", env.StringsFromEnv("CD_REPO_SERVER_PLUGIN_TAR_EXCLUSIONS", []string{}, ";"), "Globs to filter when sending tarballs to plugins.")
	command.Flags().BoolVar(&allowOutOfBoundsSymlinks, "allow-oob-symlinks", env.ParseBoolFromEnv("CD_REPO_SERVER_ALLOW_OUT_OF_BOUNDS_SYMLINKS", false), "Allow out-of-bounds symlinks in repositories (not recommended)")
	command.Flags().StringVar(&streamedManifestMaxTarSize, "streamed-manifest-max-tar-size", env.StringFromEnv("CD_REPO_SERVER_STREAMED_MANIFEST_MAX_TAR_SIZE", "100M"), "Maximum size of streamed manifest archives")
	command.Flags().StringVar(&streamedManifestMaxExtractedSize, "streamed-manifest-max-extracted-size", env.StringFromEnv("CD_REPO_SERVER_STREAMED_MANIFEST_MAX_EXTRACTED_SIZE", "1G"), "Maximum size of streamed manifest archives when extracted")
	command.Flags().StringVar(&helmManifestMaxExtractedSize, "helm-manifest-max-extracted-size", env.StringFromEnv("CD_REPO_SERVER_HELM_MANIFEST_MAX_EXTRACTED_SIZE", "1G"), "Maximum size of helm manifest archives when extracted")
	command.Flags().StringVar(&helmRegistryMaxIndexSize, "helm-registry-max-index-size", env.StringFromEnv("CD_REPO_SERVER_HELM_MANIFEST_MAX_INDEX_SIZE", "1G"), "Maximum size of registry index file")
	command.Flags().StringVar(&ociManifestMaxExtractedSize, "oci-manifest-max-extracted-size", env.StringFromEnv("CD_REPO_SERVER_OCI_MANIFEST_MAX_EXTRACTED_SIZE", "1G"), "Maximum size of oci manifest archives when extracted")
	command.Flags().BoolVar(&disableOCIManifestMaxExtractedSize, "disable-oci-manifest-max-extracted-size", env.ParseBoolFromEnv("CD_REPO_SERVER_DISABLE_OCI_MANIFEST_MAX_EXTRACTED_SIZE", false), "Disable maximum size of oci manifest archives when extracted")
	command.Flags().BoolVar(&disableManifestMaxExtractedSize, "disable-helm-manifest-max-extracted-size", env.ParseBoolFromEnv("CD_REPO_SERVER_DISABLE_HELM_MANIFEST_MAX_EXTRACTED_SIZE", false), "Disable maximum size of helm manifest archives when extracted")
	command.Flags().BoolVar(&includeHiddenDirectories, "include-hidden-directories", env.ParseBoolFromEnv("CD_REPO_SERVER_INCLUDE_HIDDEN_DIRECTORIES", false), "Include hidden directories from Git")
	command.Flags().BoolVar(&cmpUseManifestGeneratePaths, "plugin-use-manifest-generate-paths", env.ParseBoolFromEnv("CD_REPO_SERVER_PLUGIN_USE_MANIFEST_GENERATE_PATHS", false), "Pass the resources described in cd.hanzo.ai/manifest-generate-paths value to the cmpserver to generate the application manifests.")
	command.Flags().StringSliceVar(&ociMediaTypes, "oci-layer-media-types", env.StringsFromEnv("CD_REPO_SERVER_OCI_LAYER_MEDIA_TYPES", []string{"application/vnd.oci.image.layer.v1.tar", "application/vnd.oci.image.layer.v1.tar+gzip", "application/vnd.cncf.helm.chart.content.v1.tar+gzip"}, ","), "Comma separated list of allowed media types for OCI media types. This only accounts for media types within layers.")
	command.Flags().BoolVar(&enableBuiltinGitConfig, "enable-builtin-git-config", env.ParseBoolFromEnv("CD_REPO_SERVER_ENABLE_BUILTIN_GIT_CONFIG", true), "Enable builtin git configuration options that are required for correct cd-repo-server operation.")
	command.Flags().BoolVar(&disableTLS, "disable-tls", env.ParseBoolFromEnv("CD_REPO_SERVER_DISABLE_TLS", false), "Disable TLS for the repo-server gRPC endpoint")
	command.Flags().StringVar(&clientCAPath, "client-ca-path", env.StringFromEnv("CD_REPO_SERVER_CLIENT_CA_PATH", "/app/config/reposerver/mtls/client-ca.crt"), "Path to the client CA certificate file for mTLS. Defaults to the auto-mounted Secret path; mTLS is skipped if the file does not exist.")

	tlsConfigCustomizerSrc = tls.AddTLSFlagsToCmd(&command)
	cacheSrc = reposervercache.AddCacheFlagsToCmd(&command, cacheutil.Options{
		OnClientCreated: func(client *redis.Client) {
			redisClient = client
		},
	})
	return &command
}

func buildHealthCheckTLSConfig(healthCheckClientCert *ctls.Certificate, disableTLS bool) *tls.Configuration {
	cfg := &tls.Configuration{
		DisableTLS: disableTLS,
		// InsecureSkipVerify=true: it is always a localhost connection, so no reason to re-verify the server
		// certificate on every probe.
		StrictValidation: false,
	}
	if healthCheckClientCert != nil {
		cfg.ClientCertificates = []ctls.Certificate{*healthCheckClientCert}
	}
	return cfg
}