Files
tasks/tools/sql/handler.go
Tim Deeb-SwihartandGitHub be7b5514a5 Reconnect to SQL databases when connections fail (#5926)
## What changed?

Both our PostgreSQL and MySQL database backends will now automatically
reconnect to the database when certain errors occur: all errors chosen
have been experienced when testing this behavior through an AWS Aurora
RDS failover of either MySQL or PostgreSQL.

For both backends we will reconnect when we see:
- `ECONNRESET`
- `ECONNABORTED`
- `ECONNREFUSED`
- `io.EOF`
- `io.ErrUnexpectedEOF`
- `database/sql/driver.ErrBadConn`

for postgres we will also reconnect on the following SQLStates:
- `25006` read-only transaction
- `57P03` cannot connect now
- `0A000` feature not supported, but ONLY when the message is `cannot
set transaction read-write mode during recovery`

for mysql we will also reconnect when we see the following error codes:
- `1040` too many connections 
- `1792` read-only transaction (SQLstate `25006`)
- `1836` running in read-only mode

This logic is easily extensible should we discover more failure modes
over time

## Why?

We've had multiple community reports of Temporal problems during RDS
failover. One part of this is the fact that we wouldn't necessarily
reconnect; we were at the whims of our chosen SQL abstraction's
connection pooling logic.

## How did you test it?

I manually tested this functionality in the presence of repeated RDS
failovers: 

- [x] postgres12 plugin with pq driver
- [x] postgres12 plugin with pgx driver
- [x] mysql plugin

Automated testing will be added to our regular testing pipelines once
our infrastructure friends have added the support I need (it's in
progress)

## Potential risks

We're concerned there's a correctness issue in our PostgreSQL backend
that's related to our behavior during an RDS failover. If we merge this
before I figure out what's going on, we could hide the issue and make it
harder to reproduce.

## Documentation
N/A

## Is hotfix candidate?
Yes?
2024-05-17 16:59:15 -07:00

208 lines
6.3 KiB
Go

// The MIT License
//
// Copyright (c) 2020 Temporal Technologies Inc. All rights reserved.
//
// Copyright (c) 2020 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
package sql
import (
"fmt"
"net"
"net/url"
"github.com/urfave/cli"
"go.temporal.io/server/common/auth"
"go.temporal.io/server/common/config"
"go.temporal.io/server/common/log"
"go.temporal.io/server/common/log/tag"
"go.temporal.io/server/tools/common/schema"
)
// setupSchema executes the setupSchemaTask
// using the given command line arguments
// as input
func setupSchema(cli *cli.Context, logger log.Logger) error {
cfg, err := parseConnectConfig(cli)
if err != nil {
logger.Error("Unable to read config.", tag.Error(schema.NewConfigError(err.Error())))
return err
}
conn, err := NewConnection(cfg, logger)
if err != nil {
logger.Error("Unable to connect to SQL database.", tag.Error(err))
return err
}
defer conn.Close()
if err := schema.Setup(cli, conn, logger); err != nil {
logger.Error("Unable to setup SQL schema.", tag.Error(err))
return err
}
return nil
}
// updateSchema executes the updateSchemaTask
// using the given command line args as input
func updateSchema(cli *cli.Context, logger log.Logger) error {
cfg, err := parseConnectConfig(cli)
if err != nil {
logger.Error("Unable to read config.", tag.Error(schema.NewConfigError(err.Error())))
return err
}
conn, err := NewConnection(cfg, logger)
if err != nil {
logger.Error("Unable to connect to SQL database.", tag.Error(err))
return err
}
defer conn.Close()
if err := schema.Update(cli, conn, logger); err != nil {
logger.Error("Unable to update SQL schema.", tag.Error(err))
return err
}
return nil
}
// createDatabase creates a sql database
func createDatabase(cli *cli.Context, logger log.Logger) error {
cfg, err := parseConnectConfig(cli)
if err != nil {
logger.Error("Unable to read config.", tag.Error(schema.NewConfigError(err.Error())))
return err
}
defaultDb := cli.String(schema.CLIOptDefaultDb)
err = DoCreateDatabase(cfg, defaultDb, logger)
if err != nil {
logger.Error("Unable to create SQL database.", tag.Error(err))
return err
}
return nil
}
func DoCreateDatabase(cfg *config.SQL, defaultDb string, logger log.Logger) error {
dbToCreate := cfg.DatabaseName
cfg.DatabaseName = defaultDb
conn, err := NewConnection(cfg, logger)
if err != nil {
return err
}
defer conn.Close()
return conn.CreateDatabase(dbToCreate)
}
// dropDatabase drops a sql database
func dropDatabase(cli *cli.Context, logger log.Logger) error {
cfg, err := parseConnectConfig(cli)
if err != nil {
logger.Error("Unable to read config.", tag.Error(schema.NewConfigError(err.Error())))
return err
}
defaultDb := cli.String(schema.CLIOptDefaultDb)
err = DoDropDatabase(cfg, defaultDb, logger)
if err != nil {
logger.Error("Unable to drop SQL database.", tag.Error(err))
return err
}
return nil
}
func DoDropDatabase(cfg *config.SQL, defaultDb string, logger log.Logger) error {
dbToDrop := cfg.DatabaseName
cfg.DatabaseName = defaultDb
conn, err := NewConnection(cfg, logger)
if err != nil {
return err
}
err = conn.DropDatabase(dbToDrop)
if err != nil {
return err
}
conn.Close()
return nil
}
func parseConnectConfig(cli *cli.Context) (*config.SQL, error) {
cfg := new(config.SQL)
host := cli.GlobalString(schema.CLIOptEndpoint)
port := cli.GlobalInt(schema.CLIOptPort)
cfg.ConnectAddr = fmt.Sprintf("%s:%v", host, port)
cfg.User = cli.GlobalString(schema.CLIOptUser)
cfg.Password = cli.GlobalString(schema.CLIOptPassword)
cfg.DatabaseName = cli.GlobalString(schema.CLIOptDatabase)
cfg.PluginName = cli.GlobalString(schema.CLIOptPluginName)
if cfg.ConnectAttributes == nil {
cfg.ConnectAttributes = map[string]string{}
}
connectAttributesQueryString := cli.GlobalString(schema.CLIOptConnectAttributes)
if connectAttributesQueryString != "" {
values, err := url.ParseQuery(connectAttributesQueryString)
if err != nil {
return nil, fmt.Errorf("invalid connect attributes: %v", err)
}
for key, vals := range values {
// check to ensure only one value is provider per key
if len(vals) > 1 {
return nil, fmt.Errorf("invalid connect attribute %v, only 1 value allowed: %v", key, vals)
}
cfg.ConnectAttributes[key] = vals[0]
}
}
if cli.GlobalBool(schema.CLIFlagEnableTLS) {
cfg.TLS = &auth.TLS{
Enabled: true,
CertFile: cli.GlobalString(schema.CLIFlagTLSCertFile),
KeyFile: cli.GlobalString(schema.CLIFlagTLSKeyFile),
CaFile: cli.GlobalString(schema.CLIFlagTLSCaFile),
ServerName: cli.GlobalString(schema.CLIFlagTLSHostName),
EnableHostVerification: !cli.GlobalBool(schema.CLIFlagTLSDisableHostVerification),
}
}
if err := ValidateConnectConfig(cfg); err != nil {
return nil, err
}
return cfg, nil
}
// ValidateConnectConfig validates params
func ValidateConnectConfig(cfg *config.SQL) error {
host, _, err := net.SplitHostPort(cfg.ConnectAddr)
if err != nil {
return schema.NewConfigError("invalid host and port " + cfg.ConnectAddr)
}
if len(host) == 0 {
return schema.NewConfigError("missing sql endpoint argument " + flag(schema.CLIOptEndpoint))
}
if cfg.DatabaseName == "" {
return schema.NewConfigError("missing " + flag(schema.CLIOptDatabase) + " argument")
}
return nil
}
func flag(opt string) string {
return "(-" + opt + ")"
}