mirror of
https://github.com/hanzoai/porto.git
synced 2026-11-03 18:08:27 +00:00
471 lines
13 KiB
Go
471 lines
13 KiB
Go
package gserver
|
|
|
|
import (
|
|
"context"
|
|
"net"
|
|
"os"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/cockroachdb/errors"
|
|
"github.com/effective-security/porto/gserver/roles"
|
|
"github.com/effective-security/porto/pkg/discovery"
|
|
"github.com/effective-security/porto/pkg/transport"
|
|
"github.com/effective-security/porto/restserver"
|
|
"github.com/effective-security/porto/restserver/authz"
|
|
"github.com/effective-security/porto/xhttp/identity"
|
|
"github.com/effective-security/x/netutil"
|
|
"github.com/effective-security/xlog"
|
|
"github.com/effective-security/xpki/jwt"
|
|
"go.uber.org/dig"
|
|
"google.golang.org/grpc"
|
|
)
|
|
|
|
var logger = xlog.NewPackageLogger("github.com/effective-security/porto", "gserver")
|
|
|
|
// ServiceFactory builds a Service for the given server. The returned value is
|
|
// passed to dig.Container.Invoke, so it must be a function whose parameters are
|
|
// resolved from the container; the function itself should call GServer.AddService.
|
|
type ServiceFactory func(GServer) any
|
|
|
|
// Service is a named sub-service hosted by a Server. Services are created by a
|
|
// ServiceFactory and registered with GServer.AddService; a Service may
|
|
// additionally implement RouteRegistrator, GRPCRegistrator or StartSubcriber.
|
|
type Service interface {
|
|
// Name returns the unique service name used as the key in GServer.Service.
|
|
Name() string
|
|
// Close releases the service resources; it is called by Server.Close before
|
|
// the listeners are shut down.
|
|
Close()
|
|
// IsReady indicates that service is ready to serve its end-points
|
|
IsReady() bool
|
|
}
|
|
|
|
// StartSubcriber is an optional interface a Service can implement to be
|
|
// notified once the server has started.
|
|
type StartSubcriber interface {
|
|
// OnStarted is called when the server started and
|
|
// is ready to serve requests
|
|
OnStarted() error
|
|
}
|
|
|
|
// RouteRegistrator is an optional interface a Service implements to expose
|
|
// REST endpoints; it is invoked once per listener when the router is built.
|
|
type RouteRegistrator interface {
|
|
// RegisterRoute adds the service HTTP handlers to the router.
|
|
RegisterRoute(restserver.Router)
|
|
}
|
|
|
|
// GRPCRegistrator is an optional interface a Service implements to expose gRPC
|
|
// services; it is invoked once per gRPC server instance (secure and insecure).
|
|
type GRPCRegistrator interface {
|
|
// RegisterGRPC registers the service implementation on the gRPC server.
|
|
RegisterGRPC(*grpc.Server)
|
|
}
|
|
|
|
// GServer is the server handle passed to ServiceFactory functions and
|
|
// returned by Start. Services use it to register themselves and to query
|
|
// server state; it is safe for concurrent reads after Start returns.
|
|
type GServer interface {
|
|
// Name returns server name
|
|
Name() string
|
|
// Configuration of the server
|
|
Configuration() *Config
|
|
// AddService to the server
|
|
AddService(svc Service)
|
|
// Service returns service by name
|
|
Service(name string) Service
|
|
// IsReady returns true when the server is ready to serve
|
|
IsReady() bool
|
|
// StartedAt returns Time when the server has started
|
|
StartedAt() time.Time
|
|
// ListenURLs is the list of URLs that the server listens on
|
|
ListenURLs() []string
|
|
// Hostname is the hostname
|
|
Hostname() string
|
|
// LocalIP is the local IP4
|
|
LocalIP() string
|
|
// Discovery returns Discovery interface
|
|
Discovery() discovery.Discovery
|
|
// Err returns error channel
|
|
Err() <-chan error
|
|
// Close gracefully shuts down all servers/listeners and stops the TLS
|
|
// certificate reloader. It waits up to Config.Timeout.Shutdown (default
|
|
// 10s) for in-flight requests to finish, then stops the remaining ones.
|
|
// The teardown runs once: later or concurrent calls return after it
|
|
// has finished.
|
|
Close()
|
|
}
|
|
|
|
// Server is the GServer implementation returned by Start. It owns the
|
|
// listeners, the per-listener HTTP and gRPC servers, and the registered
|
|
// services. Use Start to construct it; the zero value is not usable.
|
|
type Server struct {
|
|
// Listeners are the accepted network listeners, one per unique listen
|
|
// address. They are wrappers (accept backoff, TCP keepalive), not the
|
|
// *net.TCPListener or *net.UnixListener values; use Addr and Close.
|
|
Listeners []net.Listener
|
|
|
|
ipaddr string
|
|
hostname string
|
|
// a map of contexts for the servers that serves client requests.
|
|
sctxs map[string]*serveCtx
|
|
// tlsInfo is the server TLS config shared by the TLS listeners, or nil;
|
|
// its certificate reloader runs until Close.
|
|
tlsInfo *transport.TLSInfo
|
|
|
|
di *dig.Container
|
|
name string
|
|
cfg Config
|
|
|
|
stopc chan struct{}
|
|
errc chan error
|
|
closeOnce sync.Once
|
|
startedAt time.Time
|
|
|
|
services map[string]Service
|
|
|
|
authz *authz.Provider
|
|
identity roles.IdentityProvider
|
|
trustedProxies *identity.TrustedProxies
|
|
disco discovery.Discovery
|
|
|
|
opts options
|
|
}
|
|
|
|
// Start creates the services from serviceFactories, opens the listeners from
|
|
// cfg.ListenURLs and begins serving in background goroutines, returning the
|
|
// running server. The container must provide discovery.Discovery and, when
|
|
// cfg.IdentityMap enables JWT or DPoP, a jwt.Parser. On error the created
|
|
// services are closed, the opened listeners are closed, the TLS reloader is
|
|
// stopped and the error is returned; on success the caller must eventually
|
|
// call Close. Serve errors are reported on Err.
|
|
func Start(
|
|
name string,
|
|
cfg *Config,
|
|
container *dig.Container,
|
|
serviceFactories map[string]ServiceFactory,
|
|
opts ...Option,
|
|
) (GServer, error) {
|
|
e, err := newServer(name, cfg, container, serviceFactories, opts...)
|
|
if err != nil {
|
|
if e != nil {
|
|
e.abort()
|
|
}
|
|
return nil, err
|
|
}
|
|
if err = e.resolveDependencies(container); err != nil {
|
|
e.abort()
|
|
return nil, err
|
|
}
|
|
|
|
e.serveClients()
|
|
|
|
// Register services
|
|
for _, svc := range e.services {
|
|
_ = e.disco.Register(e.Name(), svc)
|
|
}
|
|
|
|
return e, nil
|
|
}
|
|
|
|
// resolveDependencies injects discovery from the container and builds the
|
|
// identity and authorization providers from the configuration.
|
|
func (e *Server) resolveDependencies(container *dig.Container) error {
|
|
err := container.Invoke(func(d discovery.Discovery) error {
|
|
e.disco = d
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return errors.WithMessage(err, "unable to inject dependencies")
|
|
}
|
|
|
|
identityMap := e.cfg.IdentityMap
|
|
var jwtparser jwt.Parser
|
|
if identityMap != nil {
|
|
err = container.Invoke(func(jwtParser jwt.Parser) error {
|
|
jwtparser = jwtParser
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
logger.KV(xlog.ERROR, "reason", "jwt.Parser not provided", "err", err)
|
|
}
|
|
} else {
|
|
identityMap = &roles.IdentityMap{}
|
|
}
|
|
e.identity, err = roles.New(identityMap, jwtparser)
|
|
if err != nil {
|
|
return errors.WithMessage(err, "unable to create roles AuthZ")
|
|
}
|
|
|
|
authzCfg := e.cfg.Authz
|
|
if authzCfg != nil &&
|
|
(len(authzCfg.Allow) > 0 ||
|
|
len(authzCfg.AllowAny) > 0 ||
|
|
len(authzCfg.AllowAnyRole) > 0) {
|
|
e.authz, err = authz.New(authzCfg)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// abort releases a server that never started serving. No serve goroutine
|
|
// will close the serve channels, so they are closed here first and Close
|
|
// does not wait on them; Close then releases the services, the listeners
|
|
// and the TLS reloader.
|
|
func (e *Server) abort() {
|
|
for _, sctx := range e.sctxs {
|
|
sctx.closeServers()
|
|
}
|
|
e.Close()
|
|
}
|
|
|
|
func newServer(
|
|
name string,
|
|
cfg *Config,
|
|
container *dig.Container,
|
|
serviceFactories map[string]ServiceFactory,
|
|
opts ...Option,
|
|
) (*Server, error) {
|
|
trustedProxies, err := cfg.validate()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
ipaddr, err := netutil.GetLocalIP()
|
|
if err != nil {
|
|
ipaddr = "127.0.0.1"
|
|
logger.KV(xlog.ERROR, "reason", "unable_determine_ipaddr", "use", ipaddr, "err", err.Error())
|
|
}
|
|
hostname, _ := os.Hostname()
|
|
|
|
e := &Server{
|
|
ipaddr: ipaddr,
|
|
hostname: hostname,
|
|
name: name,
|
|
cfg: *cfg,
|
|
trustedProxies: trustedProxies,
|
|
di: container,
|
|
services: make(map[string]Service),
|
|
//sctxs: make(map[string]*serveCtx),
|
|
stopc: make(chan struct{}),
|
|
startedAt: time.Now(),
|
|
}
|
|
|
|
for _, o := range opts {
|
|
o.apply(&e.opts)
|
|
}
|
|
|
|
// From here on, e is returned with the error so Start can release the
|
|
// services already created and the listeners already opened.
|
|
for _, svc := range cfg.Services {
|
|
sf := serviceFactories[svc]
|
|
if sf == nil {
|
|
return e, errors.Errorf("service factory is not registered: %q", svc)
|
|
}
|
|
err = container.Invoke(sf(e))
|
|
if err != nil {
|
|
return e, errors.WithMessagef(err, "service factory failed, server=%q, service=%s",
|
|
name, svc)
|
|
}
|
|
}
|
|
|
|
logger.KV(xlog.TRACE, "status", "configuring_listeners", "server", name)
|
|
|
|
e.sctxs, e.tlsInfo, err = configureListeners(cfg)
|
|
if err != nil {
|
|
return e, err
|
|
}
|
|
|
|
for _, sctx := range e.sctxs {
|
|
e.Listeners = append(e.Listeners, sctx.listener)
|
|
}
|
|
|
|
// buffer channel so goroutines on closed connections won't wait forever
|
|
e.errc = make(chan error, len(e.Listeners)+2*len(e.sctxs))
|
|
|
|
return e, nil
|
|
}
|
|
|
|
// serveClients starts one serve goroutine per listener; serve errors are
|
|
// reported through errHandler.
|
|
func (e *Server) serveClients() {
|
|
for _, sctx := range e.sctxs {
|
|
go func(s *serveCtx) {
|
|
e.errHandler(s.serve(e, e.errHandler))
|
|
}(sctx)
|
|
}
|
|
}
|
|
|
|
func (e *Server) errHandler(err error) {
|
|
if err != nil && !strings.Contains(err.Error(), "closed") {
|
|
logger.KV(xlog.INFO, "err", err)
|
|
}
|
|
select {
|
|
case <-e.stopc:
|
|
return
|
|
default:
|
|
}
|
|
select {
|
|
case <-e.stopc:
|
|
case e.errc <- err:
|
|
}
|
|
}
|
|
|
|
// Close gracefully shuts down all servers/listeners. It waits up to
|
|
// Config.Timeout.Shutdown (default 10s) for in-flight requests to finish,
|
|
// then stops the remaining ones. It then closes the services and listeners
|
|
// and stops the TLS certificate reloader. The teardown runs once, so
|
|
// services are closed once; a later or concurrent call returns after the
|
|
// teardown has finished.
|
|
func (e *Server) Close() {
|
|
e.closeOnce.Do(e.teardown)
|
|
}
|
|
|
|
// teardown is the body of Close; closeOnce serializes it.
|
|
func (e *Server) teardown() {
|
|
logger.KV(xlog.INFO, "server", e.Name())
|
|
|
|
close(e.stopc)
|
|
|
|
// drain in-flight requests for the shutdown timeout, then stop them
|
|
timeout := e.cfg.shutdownTimeout()
|
|
for _, sctx := range e.sctxs {
|
|
for ss := range sctx.serversC {
|
|
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
|
stopServers(ctx, ss)
|
|
cancel()
|
|
}
|
|
}
|
|
|
|
for _, sctx := range e.sctxs {
|
|
sctx.cancel()
|
|
}
|
|
|
|
for _, svc := range e.services {
|
|
svc.Close()
|
|
}
|
|
|
|
for i := range e.Listeners {
|
|
if e.Listeners[i] != nil {
|
|
e.Listeners[i].Close()
|
|
}
|
|
}
|
|
|
|
if e.tlsInfo != nil {
|
|
e.tlsInfo.Close()
|
|
}
|
|
}
|
|
|
|
func stopServers(ctx context.Context, ss *servers) {
|
|
// ServeHTTP transports cannot drain through GracefulStop: their Drain
|
|
// closes the stream immediately. Let HTTP drain them before stopping gRPC.
|
|
if ss.secure {
|
|
_ = ss.http.Shutdown(ctx)
|
|
ss.grpc.Stop()
|
|
_ = ss.http.Close()
|
|
return
|
|
}
|
|
|
|
// Both protocols share the deadline. Shutdown closes idle keep-alive
|
|
// connections and drains active HTTP requests alongside the RPCs.
|
|
httpDone := make(chan struct{})
|
|
go func() {
|
|
defer close(httpDone)
|
|
_ = ss.http.Shutdown(ctx)
|
|
}()
|
|
ch := make(chan struct{})
|
|
go func() {
|
|
defer close(ch)
|
|
// close listeners to stop accepting new connections,
|
|
// will block on any existing transports
|
|
ss.grpc.GracefulStop()
|
|
}()
|
|
|
|
// wait until all pending RPCs are finished
|
|
select {
|
|
case <-ch:
|
|
case <-ctx.Done():
|
|
// took too long, manually close open transports
|
|
// e.g. watch streams
|
|
ss.grpc.Stop()
|
|
|
|
// GracefulStop can still be waiting for a handler that ignores
|
|
// cancellation. Do not delay service cleanup on that handler.
|
|
}
|
|
<-httpDone
|
|
// Shutdown leaves active HTTP connections open when its context expires.
|
|
_ = ss.http.Close()
|
|
}
|
|
|
|
// Err returns the channel on which listener/serve errors are reported. The
|
|
// channel is buffered and is never closed; errors are dropped once Close begins.
|
|
func (e *Server) Err() <-chan error { return e.errc }
|
|
|
|
// Name returns server name
|
|
func (e *Server) Name() string {
|
|
return e.name
|
|
}
|
|
|
|
// Configuration returns a pointer to the server's copy of the Config;
|
|
// mutating it after Start has no effect on already configured listeners.
|
|
func (e *Server) Configuration() *Config {
|
|
return &e.cfg
|
|
}
|
|
|
|
// AddService registers svc by its Name, replacing any service with the same
|
|
// name. It must be called from a ServiceFactory, before Start begins serving;
|
|
// the services map is not synchronized.
|
|
func (e *Server) AddService(svc Service) {
|
|
logger.KV(xlog.NOTICE, "server", e.Name(), "service", svc.Name())
|
|
|
|
e.services[svc.Name()] = svc
|
|
}
|
|
|
|
// Service returns the registered service with the given name, or nil.
|
|
func (e *Server) Service(name string) Service {
|
|
return e.services[name]
|
|
}
|
|
|
|
// IsReady returns true when every registered service reports IsReady.
|
|
func (e *Server) IsReady() bool {
|
|
for _, ss := range e.services {
|
|
if !ss.IsReady() {
|
|
logger.KV(xlog.INFO, "status", "NOT_READY", "svc", ss.Name())
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// StartedAt returns Time when the server has started
|
|
func (e *Server) StartedAt() time.Time {
|
|
return e.startedAt
|
|
}
|
|
|
|
// ListenURLs returns the configured Config.ListenURLs (not the resolved
|
|
// listener addresses).
|
|
func (e *Server) ListenURLs() []string {
|
|
return e.cfg.ListenURLs
|
|
}
|
|
|
|
// Hostname is the hostname
|
|
func (e *Server) Hostname() string {
|
|
return e.hostname
|
|
}
|
|
|
|
// LocalIP returns the local IPv4 address detected at startup, or 127.0.0.1
|
|
// when it could not be determined.
|
|
func (e *Server) LocalIP() string {
|
|
return e.ipaddr
|
|
}
|
|
|
|
// Discovery returns the discovery.Discovery injected from the container; all
|
|
// services are registered with it under the server name after Start.
|
|
func (e *Server) Discovery() discovery.Discovery {
|
|
return e.disco
|
|
}
|