mirror of
https://github.com/hanzoai/porto.git
synced 2026-11-03 18:08:27 +00:00
541 lines
15 KiB
Go
541 lines
15 KiB
Go
package gserver
|
|
|
|
import (
|
|
"context"
|
|
"crypto/tls"
|
|
"net"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/didip/tollbooth/v7"
|
|
"github.com/didip/tollbooth/v7/limiter"
|
|
"github.com/effective-security/porto/gserver/credentials"
|
|
"github.com/effective-security/porto/pkg/transport"
|
|
"github.com/effective-security/porto/restserver"
|
|
"github.com/effective-security/porto/restserver/ready"
|
|
"github.com/effective-security/porto/restserver/telemetry"
|
|
"github.com/effective-security/porto/x/slices"
|
|
"github.com/effective-security/porto/xhttp/correlation"
|
|
"github.com/effective-security/porto/xhttp/header"
|
|
"github.com/effective-security/porto/xhttp/httperror"
|
|
"github.com/effective-security/porto/xhttp/identity"
|
|
"github.com/effective-security/porto/xhttp/marshal"
|
|
"github.com/effective-security/xlog"
|
|
grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
|
|
grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
|
|
"github.com/pkg/errors"
|
|
"github.com/rs/cors"
|
|
"github.com/soheilhy/cmux"
|
|
"google.golang.org/grpc"
|
|
"google.golang.org/grpc/keepalive"
|
|
)
|
|
|
|
type serveCtx struct {
|
|
listener net.Listener
|
|
addr string
|
|
network string
|
|
secure bool
|
|
insecure bool
|
|
|
|
ctx context.Context
|
|
cancel context.CancelFunc
|
|
|
|
tlsInfo *transport.TLSInfo
|
|
|
|
cfg *Config
|
|
|
|
gopts []grpc.ServerOption
|
|
serversC chan *servers
|
|
}
|
|
|
|
type servers struct {
|
|
secure bool
|
|
grpc *grpc.Server
|
|
http *http.Server
|
|
}
|
|
|
|
func configureListeners(cfg *Config) (sctxs map[string]*serveCtx, err error) {
|
|
urls, err := cfg.ParseListenURLs()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var tlsInfo *transport.TLSInfo
|
|
if !cfg.ServerTLS.Empty() {
|
|
from := cfg.ServerTLS
|
|
clientauthType := tls.VerifyClientCertIfGiven
|
|
if from.GetClientCertAuth() {
|
|
clientauthType = tls.RequireAndVerifyClientCert
|
|
}
|
|
tlsInfo = &transport.TLSInfo{
|
|
CertFile: from.CertFile,
|
|
KeyFile: from.KeyFile,
|
|
TrustedCAFile: from.TrustedCAFile,
|
|
ClientAuthType: clientauthType,
|
|
CipherSuites: from.CipherSuites,
|
|
// CRLVerifier : TODO
|
|
}
|
|
|
|
_, err = tlsInfo.ServerTLSWithReloader()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
gopts := []grpc.ServerOption{}
|
|
if cfg.KeepAlive.MinTime > 0 {
|
|
gopts = append(gopts, grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
|
|
MinTime: cfg.KeepAlive.MinTime,
|
|
PermitWithoutStream: false,
|
|
}))
|
|
}
|
|
|
|
ka := keepalive.ServerParameters{
|
|
MaxConnectionIdle: 5 * time.Minute,
|
|
}
|
|
if cfg.KeepAlive.Interval > 0 &&
|
|
cfg.KeepAlive.Timeout > 0 {
|
|
ka.Time = cfg.KeepAlive.Interval
|
|
ka.Timeout = cfg.KeepAlive.Timeout
|
|
}
|
|
gopts = append(gopts, grpc.KeepaliveParams(ka))
|
|
|
|
sctxs = make(map[string]*serveCtx)
|
|
defer func() {
|
|
if err == nil {
|
|
return
|
|
}
|
|
// clean up on error
|
|
for _, sctx := range sctxs {
|
|
if sctx.listener != nil {
|
|
logger.KV(xlog.INFO,
|
|
"reason", "error",
|
|
"network", sctx.network,
|
|
"address", sctx.addr,
|
|
"err", err)
|
|
sctx.listener.Close()
|
|
}
|
|
}
|
|
}()
|
|
|
|
for _, u := range urls {
|
|
if u.Scheme != "" && u.Scheme != "http" && u.Scheme != "https" && u.Scheme != "unix" && u.Scheme != "unixs" {
|
|
return nil, errors.Errorf("unsupported URL scheme %q", u.Scheme)
|
|
}
|
|
|
|
if u.Scheme == "" && tlsInfo != nil {
|
|
u.Scheme = "https"
|
|
}
|
|
if (u.Scheme == "https" || u.Scheme == "unixs") && tlsInfo == nil {
|
|
return nil, errors.Errorf("TLS key/cert must be provided for the url %s with HTTPS scheme", u.String())
|
|
}
|
|
if (u.Scheme == "http" || u.Scheme == "unix") && tlsInfo != nil {
|
|
logger.KV(xlog.WARNING, "reason", "tls_without_https_scheme", "url", u.String())
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
sctx := &serveCtx{
|
|
network: "tcp",
|
|
secure: u.Scheme == "https" || u.Scheme == "unixs",
|
|
addr: u.Host,
|
|
ctx: ctx,
|
|
cancel: cancel,
|
|
cfg: cfg,
|
|
tlsInfo: tlsInfo,
|
|
gopts: gopts,
|
|
serversC: make(chan *servers, 2), // in case sctx.insecure,sctx.secure true
|
|
}
|
|
sctx.insecure = !sctx.secure
|
|
|
|
// net.Listener will rewrite ipv4 0.0.0.0 to ipv6 [::], breaking
|
|
// hosts that disable ipv6. So, use the address given by the user.
|
|
|
|
if u.Scheme == "unix" || u.Scheme == "unixs" {
|
|
sctx.network = "unix"
|
|
sctx.addr = u.Host + u.Path
|
|
}
|
|
|
|
if oldctx := sctxs[sctx.addr]; oldctx != nil {
|
|
// use existing listener
|
|
oldctx.secure = oldctx.secure || sctx.secure
|
|
oldctx.insecure = oldctx.insecure || sctx.insecure
|
|
continue
|
|
}
|
|
|
|
logger.KV(xlog.INFO,
|
|
"status", "listen",
|
|
"network", sctx.network,
|
|
"address", sctx.addr)
|
|
|
|
if sctx.listener, err = net.Listen(sctx.network, sctx.addr); err != nil {
|
|
return nil, errors.WithStack(err)
|
|
}
|
|
|
|
if sctx.network == "tcp" {
|
|
if sctx.listener, err = transport.NewKeepAliveListener(sctx.listener, sctx.network, nil); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
// TODO: register profiler, tracer, etc
|
|
|
|
sctxs[sctx.addr] = sctx
|
|
}
|
|
|
|
return sctxs, nil
|
|
}
|
|
|
|
// serve accepts incoming connections on the listener l,
|
|
// creating a new service goroutine for each. The service goroutines
|
|
// read requests and then call handler to reply to them.
|
|
func (sctx *serveCtx) serve(s *Server, errHandler func(error)) (err error) {
|
|
//<-s.ReadyNotify()
|
|
|
|
logger.KV(xlog.INFO, "status", "ready_to_serve", "service", s.Name(), "network", sctx.network, "address", sctx.addr)
|
|
|
|
var gsSecure *grpc.Server
|
|
var gsInsecure *grpc.Server
|
|
|
|
defer func() {
|
|
if err == nil {
|
|
return
|
|
}
|
|
if gsSecure != nil {
|
|
gsSecure.Stop()
|
|
}
|
|
if gsInsecure != nil {
|
|
gsInsecure.Stop()
|
|
}
|
|
}()
|
|
|
|
router := restRouter(s)
|
|
|
|
m := cmux.New(sctx.listener)
|
|
|
|
if sctx.insecure {
|
|
gsInsecure = grpcServer(s, nil, sctx.gopts...)
|
|
grpcL := m.Match(cmux.HTTP2())
|
|
go func() { errHandler(gsInsecure.Serve(grpcL)) }()
|
|
|
|
handler := router.Handler()
|
|
handler = configureHandlers(s, handler)
|
|
// rate limit will be first
|
|
handler = configureRateLimiter(s.cfg.RateLimit, handler)
|
|
|
|
srv := &http.Server{
|
|
Handler: handler,
|
|
//ErrorLog: logger, // do not log user error
|
|
}
|
|
|
|
httpL := m.Match(cmux.HTTP1())
|
|
go func() { errHandler(srv.Serve(httpL)) }()
|
|
|
|
sctx.serversC <- &servers{grpc: gsInsecure, http: srv}
|
|
|
|
logger.KV(xlog.WARNING, "reason", "insecure", "service", s.Name(), "address", sctx.addr)
|
|
}
|
|
|
|
if sctx.secure {
|
|
gsSecure = grpcServer(s, sctx.tlsInfo.Config(), sctx.gopts...)
|
|
handler := router.Handler()
|
|
handler = configureHandlers(s, handler)
|
|
|
|
// mux between http and grpc
|
|
handler = sctx.grpcHandlerFunc(gsSecure, handler)
|
|
// rate limit will be first
|
|
handler = configureRateLimiter(s.cfg.RateLimit, handler)
|
|
|
|
srv := &http.Server{
|
|
Handler: handler,
|
|
TLSConfig: sctx.tlsInfo.Config(),
|
|
//ErrorLog: logger, // do not log user error
|
|
}
|
|
grpcL, err := transport.NewTLSListener(m.Match(cmux.Any()), sctx.tlsInfo)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
go func() { errHandler(srv.Serve(grpcL)) }()
|
|
|
|
sctx.serversC <- &servers{secure: true, grpc: gsSecure, http: srv}
|
|
}
|
|
|
|
logger.KV(xlog.INFO, "status", "serving", "service", s.Name(), "address", sctx.listener.Addr().String(), "secure", sctx.secure, "insecure", sctx.insecure)
|
|
|
|
close(sctx.serversC)
|
|
|
|
// Serve starts multiplexing the listener.
|
|
// Serve blocks and perhaps should be invoked concurrently within a go routine.
|
|
return m.Serve()
|
|
}
|
|
|
|
func configureRateLimiter(cfg *RateLimit, handler http.Handler) http.Handler {
|
|
if !cfg.GetEnabled() {
|
|
return handler
|
|
}
|
|
logger.KV(xlog.NOTICE, "RateLimit", "enabled")
|
|
|
|
ttl := cfg.ExpirationTTL
|
|
if ttl == 0 {
|
|
ttl = 10 * time.Minute
|
|
}
|
|
ops := limiter.ExpirableOptions{
|
|
DefaultExpirationTTL: ttl,
|
|
}
|
|
|
|
lmt := tollbooth.NewLimiter(float64(cfg.RequestsPerSecond), &ops)
|
|
if len(cfg.HeadersIPLookups) > 0 {
|
|
lmt.SetIPLookups(cfg.HeadersIPLookups)
|
|
}
|
|
if len(cfg.Metods) > 0 {
|
|
lmt.SetMethods(cfg.Metods)
|
|
}
|
|
|
|
return tollbooth.LimitHandler(lmt, handler)
|
|
}
|
|
|
|
func configureHandlers(s *Server, handler http.Handler) http.Handler {
|
|
// NOTE: the handlers are executed in the reverse order
|
|
// therefore configure additional first
|
|
for _, other := range s.opts.handlers {
|
|
handler = other(handler)
|
|
}
|
|
|
|
// service ready
|
|
handler = ready.NewServiceStatusVerifier(s, handler)
|
|
|
|
var err error
|
|
// authz
|
|
if s.authz != nil {
|
|
handler, err = s.authz.NewHandler(handler)
|
|
if err != nil {
|
|
logger.Panicf("failed to create authz handler: %+v", err)
|
|
}
|
|
}
|
|
|
|
// logging wrapper
|
|
var opts []telemetry.Option
|
|
if len(s.cfg.SkipLogPaths) > 0 {
|
|
opts = append(opts, telemetry.WithLoggerSkipPaths(s.cfg.SkipLogPaths))
|
|
}
|
|
handler = telemetry.NewRequestLogger(handler, time.Millisecond, logger, opts...)
|
|
|
|
// metrics wrapper
|
|
handler = telemetry.NewRequestMetrics(handler)
|
|
|
|
// role/contextID wrapper
|
|
handler = identity.NewContextHandler(handler, s.identity.IdentityFromRequest)
|
|
|
|
if s.cfg.CORS.GetEnabled() {
|
|
logger.KV(xlog.NOTICE, "server", s.name, "CORS", "enabled")
|
|
co := cors.New(cors.Options{
|
|
AllowedOrigins: s.cfg.CORS.AllowedOrigins,
|
|
//AllowOriginFunc: s.cfg.CORS.AllowOriginFunc,
|
|
//AllowOriginRequestFunc: s.cfg.CORS.AllowOriginRequestFunc,
|
|
AllowedMethods: s.cfg.CORS.AllowedMethods,
|
|
AllowedHeaders: s.cfg.CORS.AllowedHeaders,
|
|
ExposedHeaders: s.cfg.CORS.ExposedHeaders,
|
|
MaxAge: s.cfg.CORS.MaxAge,
|
|
AllowCredentials: s.cfg.CORS.GetAllowCredentials(),
|
|
OptionsPassthrough: s.cfg.CORS.GetOptionsPassthrough(),
|
|
Debug: s.cfg.CORS.GetDebug(),
|
|
})
|
|
handler = co.Handler(handler)
|
|
}
|
|
|
|
// Add correlationID
|
|
handler = correlation.NewHandler(handler)
|
|
|
|
return handler
|
|
}
|
|
|
|
func restRouter(s *Server) restserver.Router {
|
|
router := restserver.NewRouter(notFoundHandler)
|
|
|
|
for name, svc := range s.services {
|
|
if registrator, ok := svc.(RouteRegistrator); ok {
|
|
logger.KV(xlog.INFO, "status", "RouteRegistrator", "server", s.Name(), "service", name)
|
|
|
|
registrator.RegisterRoute(router)
|
|
} else {
|
|
logger.KV(xlog.INFO, "status", "not_supported_RouteRegistrator", "server", s.Name(), "service", name)
|
|
}
|
|
}
|
|
|
|
return router
|
|
}
|
|
|
|
func grpcServer(s *Server, tls *tls.Config, gopts ...grpc.ServerOption) *grpc.Server {
|
|
var opts []grpc.ServerOption
|
|
//opts = append(opts, grpc.CustomCodec(&codec{}))
|
|
|
|
if tls != nil {
|
|
bundle := credentials.NewBundle(credentials.Config{TLSConfig: tls})
|
|
opts = append(opts, grpc.Creds(bundle.TransportCredentials()))
|
|
}
|
|
|
|
chainUnaryInterceptors := []grpc.UnaryServerInterceptor{
|
|
correlation.NewAuthUnaryInterceptor(),
|
|
s.newLogUnaryInterceptor(),
|
|
identity.NewAuthUnaryInterceptor(s.identity.IdentityFromContext),
|
|
s.authz.NewUnaryInterceptor(),
|
|
grpc_prometheus.UnaryServerInterceptor,
|
|
}
|
|
if len(s.opts.unary) > 0 {
|
|
chainUnaryInterceptors = append(chainUnaryInterceptors, s.opts.unary...)
|
|
}
|
|
|
|
chainStreamInterceptors := []grpc.StreamServerInterceptor{
|
|
newStreamInterceptor(s),
|
|
grpc_prometheus.StreamServerInterceptor,
|
|
}
|
|
if len(s.opts.stream) > 0 {
|
|
chainStreamInterceptors = append(chainStreamInterceptors, s.opts.stream...)
|
|
}
|
|
|
|
opts = append(opts, grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(chainUnaryInterceptors...)))
|
|
opts = append(opts, grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(chainStreamInterceptors...)))
|
|
|
|
grpcServer := grpc.NewServer(append(opts, gopts...)...)
|
|
|
|
for name, svc := range s.services {
|
|
if registrator, ok := svc.(GRPCRegistrator); ok {
|
|
logger.KV(xlog.INFO, "status", "RegisterGRPC", "server", s.Name(), "service", name)
|
|
|
|
registrator.RegisterGRPC(grpcServer)
|
|
} else {
|
|
logger.KV(xlog.INFO, "status", "not_supported_RegisterGRPC", "server", s.Name(), "service", name)
|
|
}
|
|
}
|
|
|
|
return grpcServer
|
|
}
|
|
|
|
// grpcHandlerFunc returns an http.Handler that delegates to grpcServer on incoming gRPC
|
|
// connections or otherHandler otherwise. Given in gRPC docs.
|
|
func (sctx *serveCtx) grpcHandlerFunc(grpcServer *grpc.Server, otherHandler http.Handler) http.Handler {
|
|
if otherHandler == nil {
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
grpcServer.ServeHTTP(w, r)
|
|
})
|
|
}
|
|
|
|
var allowedOrigins []string
|
|
exposedHeaders := ""
|
|
if sctx.cfg.CORS != nil {
|
|
if len(sctx.cfg.CORS.AllowedOrigins) > 0 && sctx.cfg.CORS.AllowedOrigins[0] != "*" {
|
|
allowedOrigins = sctx.cfg.CORS.AllowedOrigins
|
|
}
|
|
if len(sctx.cfg.CORS.ExposedHeaders) > 0 {
|
|
exposedHeaders = strings.Join(sctx.cfg.CORS.ExposedHeaders, ",")
|
|
}
|
|
}
|
|
|
|
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
ct := r.Header.Get(header.ContentType)
|
|
if strings.HasPrefix(ct, header.ApplicationGRPC) {
|
|
origin := r.Header.Get("Origin")
|
|
grpcWeb := ct == header.ApplicationGRPCWebProto
|
|
wh := w.Header()
|
|
if grpcWeb {
|
|
r.Header.Set(header.ContentType, header.ApplicationGRPC)
|
|
if origin != "" {
|
|
if len(allowedOrigins) > 0 && !slices.ContainsString(allowedOrigins, origin) {
|
|
logger.ContextKV(r.Context(), xlog.INFO,
|
|
"reason", "cors_not_allowed",
|
|
"method", r.Method,
|
|
"ct", ct,
|
|
"remote", r.RemoteAddr,
|
|
"agent", r.UserAgent(),
|
|
"content-type", r.Header.Get(header.ContentType),
|
|
"accept", r.Header.Get(header.Accept),
|
|
"url", r.URL.String())
|
|
return
|
|
}
|
|
if len(allowedOrigins) > 0 {
|
|
wh.Set("Access-Control-Allow-Origin", origin)
|
|
} else {
|
|
wh.Set("Access-Control-Allow-Origin", "*")
|
|
}
|
|
}
|
|
if exposedHeaders != "" {
|
|
wh.Set("Access-Control-Expose-Headers", exposedHeaders)
|
|
}
|
|
wh.Set(header.ContentType, header.ApplicationGRPC)
|
|
|
|
w = &proxyWriter{
|
|
rw: w,
|
|
}
|
|
}
|
|
if sctx.cfg.DebugLogs {
|
|
logger.ContextKV(r.Context(), xlog.DEBUG,
|
|
"method", r.Method,
|
|
"ct", ct,
|
|
"remote", r.RemoteAddr,
|
|
"agent", r.UserAgent(),
|
|
"content-type", r.Header.Get(header.ContentType),
|
|
"accept", r.Header.Get(header.Accept),
|
|
"content-length", r.ContentLength,
|
|
"proto_ver_minor", r.ProtoMinor,
|
|
"proto_ver_major", r.ProtoMajor,
|
|
"url", r.URL.String())
|
|
}
|
|
grpcServer.ServeHTTP(w, r)
|
|
if grpcWeb && sctx.cfg.DebugLogs {
|
|
logger.ContextKV(r.Context(), xlog.DEBUG,
|
|
"method", r.Method,
|
|
"headers", wh)
|
|
}
|
|
} else {
|
|
if sctx.cfg.DebugLogs && r.URL.Path != "/healthz" {
|
|
logger.ContextKV(r.Context(), xlog.DEBUG,
|
|
"handle", "otherHandler",
|
|
"ct", ct,
|
|
"remote", r.RemoteAddr,
|
|
"agent", r.UserAgent(),
|
|
"content-type", r.Header.Get(header.ContentType),
|
|
"accept", r.Header.Get(header.Accept),
|
|
"content-length", r.ContentLength,
|
|
"method", r.Method,
|
|
"url", r.URL.String(),
|
|
"proto_ver_minor", r.ProtoMinor,
|
|
"proto_ver_major", r.ProtoMajor)
|
|
}
|
|
otherHandler.ServeHTTP(w, r)
|
|
}
|
|
})
|
|
|
|
return handler
|
|
}
|
|
|
|
func notFoundHandler(w http.ResponseWriter, r *http.Request) {
|
|
marshal.WriteJSON(w, r, httperror.NotFound(r.URL.Path))
|
|
}
|
|
|
|
// the proxy is a workaround to disable premature Flush for Grpc-Web
|
|
type proxyWriter struct {
|
|
rw http.ResponseWriter
|
|
}
|
|
|
|
// Header proxy
|
|
func (p *proxyWriter) Header() http.Header {
|
|
return p.rw.Header()
|
|
}
|
|
|
|
// Write proxy
|
|
func (p *proxyWriter) Write(data []byte) (int, error) {
|
|
return p.rw.Write(data)
|
|
}
|
|
|
|
// WriteHeader proxy
|
|
func (p *proxyWriter) WriteHeader(statusCode int) {
|
|
p.rw.WriteHeader(statusCode)
|
|
}
|
|
|
|
// Flush proxy
|
|
func (p *proxyWriter) Flush() {
|
|
// do nothing
|
|
// Looks like a bug in
|
|
// func (ht *serverHandlerTransport) WriteStatus(s *Stream, st *status.Status)
|
|
logger.KV(xlog.DEBUG, "reason", "not_supported")
|
|
}
|