Files

471 lines
13 KiB
Go

package gserver
import (
"context"
"net"
"os"
"strings"
"sync"
"time"
"github.com/cockroachdb/errors"
"github.com/effective-security/porto/gserver/roles"
"github.com/effective-security/porto/pkg/discovery"
"github.com/effective-security/porto/pkg/transport"
"github.com/effective-security/porto/restserver"
"github.com/effective-security/porto/restserver/authz"
"github.com/effective-security/porto/xhttp/identity"
"github.com/effective-security/x/netutil"
"github.com/effective-security/xlog"
"github.com/effective-security/xpki/jwt"
"go.uber.org/dig"
"google.golang.org/grpc"
)
var logger = xlog.NewPackageLogger("github.com/effective-security/porto", "gserver")
// ServiceFactory builds a Service for the given server. The returned value is
// passed to dig.Container.Invoke, so it must be a function whose parameters are
// resolved from the container; the function itself should call GServer.AddService.
type ServiceFactory func(GServer) any
// Service is a named sub-service hosted by a Server. Services are created by a
// ServiceFactory and registered with GServer.AddService; a Service may
// additionally implement RouteRegistrator, GRPCRegistrator or StartSubcriber.
type Service interface {
// Name returns the unique service name used as the key in GServer.Service.
Name() string
// Close releases the service resources; it is called by Server.Close before
// the listeners are shut down.
Close()
// IsReady indicates that service is ready to serve its end-points
IsReady() bool
}
// StartSubcriber is an optional interface a Service can implement to be
// notified once the server has started.
type StartSubcriber interface {
// OnStarted is called when the server started and
// is ready to serve requests
OnStarted() error
}
// RouteRegistrator is an optional interface a Service implements to expose
// REST endpoints; it is invoked once per listener when the router is built.
type RouteRegistrator interface {
// RegisterRoute adds the service HTTP handlers to the router.
RegisterRoute(restserver.Router)
}
// GRPCRegistrator is an optional interface a Service implements to expose gRPC
// services; it is invoked once per gRPC server instance (secure and insecure).
type GRPCRegistrator interface {
// RegisterGRPC registers the service implementation on the gRPC server.
RegisterGRPC(*grpc.Server)
}
// GServer is the server handle passed to ServiceFactory functions and
// returned by Start. Services use it to register themselves and to query
// server state; it is safe for concurrent reads after Start returns.
type GServer interface {
// Name returns server name
Name() string
// Configuration of the server
Configuration() *Config
// AddService to the server
AddService(svc Service)
// Service returns service by name
Service(name string) Service
// IsReady returns true when the server is ready to serve
IsReady() bool
// StartedAt returns Time when the server has started
StartedAt() time.Time
// ListenURLs is the list of URLs that the server listens on
ListenURLs() []string
// Hostname is the hostname
Hostname() string
// LocalIP is the local IP4
LocalIP() string
// Discovery returns Discovery interface
Discovery() discovery.Discovery
// Err returns error channel
Err() <-chan error
// Close gracefully shuts down all servers/listeners and stops the TLS
// certificate reloader. It waits up to Config.Timeout.Shutdown (default
// 10s) for in-flight requests to finish, then stops the remaining ones.
// The teardown runs once: later or concurrent calls return after it
// has finished.
Close()
}
// Server is the GServer implementation returned by Start. It owns the
// listeners, the per-listener HTTP and gRPC servers, and the registered
// services. Use Start to construct it; the zero value is not usable.
type Server struct {
// Listeners are the accepted network listeners, one per unique listen
// address. They are wrappers (accept backoff, TCP keepalive), not the
// *net.TCPListener or *net.UnixListener values; use Addr and Close.
Listeners []net.Listener
ipaddr string
hostname string
// a map of contexts for the servers that serves client requests.
sctxs map[string]*serveCtx
// tlsInfo is the server TLS config shared by the TLS listeners, or nil;
// its certificate reloader runs until Close.
tlsInfo *transport.TLSInfo
di *dig.Container
name string
cfg Config
stopc chan struct{}
errc chan error
closeOnce sync.Once
startedAt time.Time
services map[string]Service
authz *authz.Provider
identity roles.IdentityProvider
trustedProxies *identity.TrustedProxies
disco discovery.Discovery
opts options
}
// Start creates the services from serviceFactories, opens the listeners from
// cfg.ListenURLs and begins serving in background goroutines, returning the
// running server. The container must provide discovery.Discovery and, when
// cfg.IdentityMap enables JWT or DPoP, a jwt.Parser. On error the created
// services are closed, the opened listeners are closed, the TLS reloader is
// stopped and the error is returned; on success the caller must eventually
// call Close. Serve errors are reported on Err.
func Start(
name string,
cfg *Config,
container *dig.Container,
serviceFactories map[string]ServiceFactory,
opts ...Option,
) (GServer, error) {
e, err := newServer(name, cfg, container, serviceFactories, opts...)
if err != nil {
if e != nil {
e.abort()
}
return nil, err
}
if err = e.resolveDependencies(container); err != nil {
e.abort()
return nil, err
}
e.serveClients()
// Register services
for _, svc := range e.services {
_ = e.disco.Register(e.Name(), svc)
}
return e, nil
}
// resolveDependencies injects discovery from the container and builds the
// identity and authorization providers from the configuration.
func (e *Server) resolveDependencies(container *dig.Container) error {
err := container.Invoke(func(d discovery.Discovery) error {
e.disco = d
return nil
})
if err != nil {
return errors.WithMessage(err, "unable to inject dependencies")
}
identityMap := e.cfg.IdentityMap
var jwtparser jwt.Parser
if identityMap != nil {
err = container.Invoke(func(jwtParser jwt.Parser) error {
jwtparser = jwtParser
return nil
})
if err != nil {
logger.KV(xlog.ERROR, "reason", "jwt.Parser not provided", "err", err)
}
} else {
identityMap = &roles.IdentityMap{}
}
e.identity, err = roles.New(identityMap, jwtparser)
if err != nil {
return errors.WithMessage(err, "unable to create roles AuthZ")
}
authzCfg := e.cfg.Authz
if authzCfg != nil &&
(len(authzCfg.Allow) > 0 ||
len(authzCfg.AllowAny) > 0 ||
len(authzCfg.AllowAnyRole) > 0) {
e.authz, err = authz.New(authzCfg)
if err != nil {
return err
}
}
return nil
}
// abort releases a server that never started serving. No serve goroutine
// will close the serve channels, so they are closed here first and Close
// does not wait on them; Close then releases the services, the listeners
// and the TLS reloader.
func (e *Server) abort() {
for _, sctx := range e.sctxs {
sctx.closeServers()
}
e.Close()
}
func newServer(
name string,
cfg *Config,
container *dig.Container,
serviceFactories map[string]ServiceFactory,
opts ...Option,
) (*Server, error) {
trustedProxies, err := cfg.validate()
if err != nil {
return nil, err
}
ipaddr, err := netutil.GetLocalIP()
if err != nil {
ipaddr = "127.0.0.1"
logger.KV(xlog.ERROR, "reason", "unable_determine_ipaddr", "use", ipaddr, "err", err.Error())
}
hostname, _ := os.Hostname()
e := &Server{
ipaddr: ipaddr,
hostname: hostname,
name: name,
cfg: *cfg,
trustedProxies: trustedProxies,
di: container,
services: make(map[string]Service),
//sctxs: make(map[string]*serveCtx),
stopc: make(chan struct{}),
startedAt: time.Now(),
}
for _, o := range opts {
o.apply(&e.opts)
}
// From here on, e is returned with the error so Start can release the
// services already created and the listeners already opened.
for _, svc := range cfg.Services {
sf := serviceFactories[svc]
if sf == nil {
return e, errors.Errorf("service factory is not registered: %q", svc)
}
err = container.Invoke(sf(e))
if err != nil {
return e, errors.WithMessagef(err, "service factory failed, server=%q, service=%s",
name, svc)
}
}
logger.KV(xlog.TRACE, "status", "configuring_listeners", "server", name)
e.sctxs, e.tlsInfo, err = configureListeners(cfg)
if err != nil {
return e, err
}
for _, sctx := range e.sctxs {
e.Listeners = append(e.Listeners, sctx.listener)
}
// buffer channel so goroutines on closed connections won't wait forever
e.errc = make(chan error, len(e.Listeners)+2*len(e.sctxs))
return e, nil
}
// serveClients starts one serve goroutine per listener; serve errors are
// reported through errHandler.
func (e *Server) serveClients() {
for _, sctx := range e.sctxs {
go func(s *serveCtx) {
e.errHandler(s.serve(e, e.errHandler))
}(sctx)
}
}
func (e *Server) errHandler(err error) {
if err != nil && !strings.Contains(err.Error(), "closed") {
logger.KV(xlog.INFO, "err", err)
}
select {
case <-e.stopc:
return
default:
}
select {
case <-e.stopc:
case e.errc <- err:
}
}
// Close gracefully shuts down all servers/listeners. It waits up to
// Config.Timeout.Shutdown (default 10s) for in-flight requests to finish,
// then stops the remaining ones. It then closes the services and listeners
// and stops the TLS certificate reloader. The teardown runs once, so
// services are closed once; a later or concurrent call returns after the
// teardown has finished.
func (e *Server) Close() {
e.closeOnce.Do(e.teardown)
}
// teardown is the body of Close; closeOnce serializes it.
func (e *Server) teardown() {
logger.KV(xlog.INFO, "server", e.Name())
close(e.stopc)
// drain in-flight requests for the shutdown timeout, then stop them
timeout := e.cfg.shutdownTimeout()
for _, sctx := range e.sctxs {
for ss := range sctx.serversC {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
stopServers(ctx, ss)
cancel()
}
}
for _, sctx := range e.sctxs {
sctx.cancel()
}
for _, svc := range e.services {
svc.Close()
}
for i := range e.Listeners {
if e.Listeners[i] != nil {
e.Listeners[i].Close()
}
}
if e.tlsInfo != nil {
e.tlsInfo.Close()
}
}
func stopServers(ctx context.Context, ss *servers) {
// ServeHTTP transports cannot drain through GracefulStop: their Drain
// closes the stream immediately. Let HTTP drain them before stopping gRPC.
if ss.secure {
_ = ss.http.Shutdown(ctx)
ss.grpc.Stop()
_ = ss.http.Close()
return
}
// Both protocols share the deadline. Shutdown closes idle keep-alive
// connections and drains active HTTP requests alongside the RPCs.
httpDone := make(chan struct{})
go func() {
defer close(httpDone)
_ = ss.http.Shutdown(ctx)
}()
ch := make(chan struct{})
go func() {
defer close(ch)
// close listeners to stop accepting new connections,
// will block on any existing transports
ss.grpc.GracefulStop()
}()
// wait until all pending RPCs are finished
select {
case <-ch:
case <-ctx.Done():
// took too long, manually close open transports
// e.g. watch streams
ss.grpc.Stop()
// GracefulStop can still be waiting for a handler that ignores
// cancellation. Do not delay service cleanup on that handler.
}
<-httpDone
// Shutdown leaves active HTTP connections open when its context expires.
_ = ss.http.Close()
}
// Err returns the channel on which listener/serve errors are reported. The
// channel is buffered and is never closed; errors are dropped once Close begins.
func (e *Server) Err() <-chan error { return e.errc }
// Name returns server name
func (e *Server) Name() string {
return e.name
}
// Configuration returns a pointer to the server's copy of the Config;
// mutating it after Start has no effect on already configured listeners.
func (e *Server) Configuration() *Config {
return &e.cfg
}
// AddService registers svc by its Name, replacing any service with the same
// name. It must be called from a ServiceFactory, before Start begins serving;
// the services map is not synchronized.
func (e *Server) AddService(svc Service) {
logger.KV(xlog.NOTICE, "server", e.Name(), "service", svc.Name())
e.services[svc.Name()] = svc
}
// Service returns the registered service with the given name, or nil.
func (e *Server) Service(name string) Service {
return e.services[name]
}
// IsReady returns true when every registered service reports IsReady.
func (e *Server) IsReady() bool {
for _, ss := range e.services {
if !ss.IsReady() {
logger.KV(xlog.INFO, "status", "NOT_READY", "svc", ss.Name())
return false
}
}
return true
}
// StartedAt returns Time when the server has started
func (e *Server) StartedAt() time.Time {
return e.startedAt
}
// ListenURLs returns the configured Config.ListenURLs (not the resolved
// listener addresses).
func (e *Server) ListenURLs() []string {
return e.cfg.ListenURLs
}
// Hostname is the hostname
func (e *Server) Hostname() string {
return e.hostname
}
// LocalIP returns the local IPv4 address detected at startup, or 127.0.0.1
// when it could not be determined.
func (e *Server) LocalIP() string {
return e.ipaddr
}
// Discovery returns the discovery.Discovery injected from the container; all
// services are registered with it under the server name after Start.
func (e *Server) Discovery() discovery.Discovery {
return e.disco
}