hanzoai/oxy
2
0
Fork 0
mirror of https://github.com/hanzoai/oxy.git synced 2026-09-30 23:42:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Recovered circuit breaker tests

Browse Source
This commit is contained in:
Alexander Klizhentas
2014-12-11 22:40:11 -08:00
parent f393f65290
commit 71e77b51bd
3 changed files with 77 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files
circuitbreaker/cbreaker_test.go
+60 -133
View File
@@ -1,9 +1,11 @@
package circuitbreaker
import (
// "fmt"
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"net/url"
"testing"
"time"
@@ -116,79 +118,74 @@ func (s *CBSuite) TestFullCycle(c *C) {
c.Assert(re.StatusCode, Equals, http.StatusOK)
}
/*
func (s *CBSuite) TestRedirect(c *C) {
cb := s.new(c,
triggerNetRatio,
fallbackRedirect,
Options{
FallbackDuration: 10 * time.Second,
RecoveryDuration: 10 * time.Second,
})
handler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
w.Write([]byte("hello"))
})
req := makeRequest(O{})
cb.metrics = statsNetErrors(0.6)
re, err := cb.ProcessRequest(req)
c.Assert(re, IsNil)
cb, err := New(handler, triggerNetRatio, Clock(s.clock), Fallback(fallbackRedirect))
c.Assert(err, IsNil)
cb.ProcessResponse(req, req.Attempts[0])
c.Assert(cb.state, Equals, cbState(stateTripped))
srv := httptest.NewServer(cb)
defer srv.Close()
re, err = cb.ProcessRequest(req)
c.Assert(re, IsNil)
c.Assert(err, DeepEquals, &errors.RedirectError{URL: netutils.MustParseUrl("http://localhost:5000")})
cb.metrics = statsNetErrors(0.6)
re, _, err := testutils.Get(srv.URL)
c.Assert(err, IsNil)
client := &http.Client{
CheckRedirect: func(req *http.Request, via []*http.Request) error {
return fmt.Errorf("no redirects")
},
}
re, err = client.Get(srv.URL)
c.Assert(err, NotNil)
c.Assert(re.StatusCode, Equals, http.StatusFound)
c.Assert(re.Header.Get("Location"), Equals, "http://localhost:5000")
}
func (s *CBSuite) TestTriggerDuringRecovery(c *C) {
cb := s.new(c,
triggerNetRatio,
fallbackResponse,
Options{
FallbackDuration: 10 * time.Second,
RecoveryDuration: 10 * time.Second,
CheckPeriod: time.Microsecond,
})
handler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
w.Write([]byte("hello"))
})
req := makeRequest(O{id: 8, stats: statsNetErrors(0.6)})
re, err := cb.ProcessRequest(req)
c.Assert(re, IsNil)
cb, err := New(handler, triggerNetRatio, Clock(s.clock), CheckPeriod(time.Microsecond))
c.Assert(err, IsNil)
srv := httptest.NewServer(cb)
defer srv.Close()
cb.metrics = statsNetErrors(0.6)
cb.ProcessResponse(req, req.Attempts[0])
c.Assert(cb.state, Equals, cbState(stateTripped))
re, err = cb.ProcessRequest(req)
c.Assert(re, NotNil)
re, _, err := testutils.Get(srv.URL)
c.Assert(err, IsNil)
c.Assert(re.StatusCode, Equals, http.StatusBadRequest)
c.Assert(cb.state, Equals, cbState(stateTripped))
// We should be in recovering state by now
okReq := makeRequest(O{stats: statsOK()})
s.advanceTime(10*time.Second + time.Millisecond)
re, err = cb.ProcessRequest(okReq)
c.Assert(re, NotNil)
re, _, err = testutils.Get(srv.URL)
c.Assert(err, IsNil)
c.Assert(re.StatusCode, Equals, http.StatusBadRequest)
c.Assert(re.StatusCode, Equals, http.StatusServiceUnavailable)
c.Assert(cb.state, Equals, cbState(stateRecovering))
cb.ProcessResponse(okReq, okReq.Attempts[0])
// We have triggered it during recovery state and are going back to triggered state
s.advanceTime(time.Millisecond)
badReq := makeRequest(O{})
// We have matched error condition during recovery state and are going back to tripped state
s.advanceTime(5 * time.Second)
cb.metrics = statsNetErrors(0.6)
cb.ProcessRequest(badReq)
cb.ProcessResponse(badReq, badReq.Attempts[0])
allowed := 0
for i := 0; i < 100; i++ {
re, _, err = testutils.Get(srv.URL)
if re.StatusCode == http.StatusOK && err == nil {
allowed++
}
}
c.Assert(allowed, Not(Equals), 0)
c.Assert(cb.state, Equals, cbState(stateTripped))
c.Assert(cb.until, Equals, s.tm.UtcNow().Add(10*time.Second))
}
func (s *CBSuite) TestSideEffects(c *C) {
srv1Chan := make(chan *http.Request, 1)
var srv1Body []byte
srv1 := testutils.NewTestServer(func(w http.ResponseWriter, r *http.Request) {
srv1 := testutils.NewHandler(func(w http.ResponseWriter, r *http.Request) {
b, err := ioutil.ReadAll(r.Body)
c.Assert(err, IsNil)
srv1Body = b
@@ -198,7 +195,7 @@ func (s *CBSuite) TestSideEffects(c *C) {
defer srv1.Close()
srv2Chan := make(chan *http.Request, 1)
srv2 := testutils.NewTestServer(func(w http.ResponseWriter, r *http.Request) {
srv2 := testutils.NewHandler(func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("srv2"))
r.ParseForm()
srv2Chan <- r
@@ -222,23 +219,20 @@ func (s *CBSuite) TestSideEffects(c *C) {
})
c.Assert(err, IsNil)
cb := s.new(c,
triggerNetRatio,
fallbackResponse,
Options{
FallbackDuration: 10 * time.Second,
RecoveryDuration: 10 * time.Second,
OnTripped: onTripped,
OnStandby: onStandby,
})
handler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
w.Write([]byte("hello"))
})
req := makeRequest(O{id: 8})
cb.metrics = statsNetErrors(0.6)
re, err := cb.ProcessRequest(req)
c.Assert(re, IsNil)
cb, err := New(handler, triggerNetRatio, Clock(s.clock), CheckPeriod(time.Microsecond), OnTripped(onTripped), OnStandby(onStandby))
c.Assert(err, IsNil)
cb.ProcessResponse(req, req.Attempts[0])
srv := httptest.NewServer(cb)
defer srv.Close()
cb.metrics = statsNetErrors(0.6)
_, _, err = testutils.Get(srv.URL)
c.Assert(err, IsNil)
c.Assert(cb.state, Equals, cbState(stateTripped))
select {
@@ -252,16 +246,14 @@ func (s *CBSuite) TestSideEffects(c *C) {
}
// Transition to recovering state
okReq := makeRequest(O{stats: statsOK()})
s.advanceTime(10*time.Second + time.Millisecond)
cb.ProcessRequest(okReq)
cb.metrics = statsOK()
testutils.Get(srv.URL)
c.Assert(cb.state, Equals, cbState(stateRecovering))
cb.ProcessResponse(okReq, okReq.Attempts[0])
// Going back to standby
s.advanceTime(10*time.Second + time.Millisecond)
cb.ProcessRequest(okReq)
cb.ProcessResponse(okReq, req.Attempts[0])
testutils.Get(srv.URL)
c.Assert(cb.state, Equals, cbState(stateStandby))
select {
@@ -274,71 +266,6 @@ func (s *CBSuite) TestSideEffects(c *C) {
}
}
func (s *CBSuite) TestInvalidParams(c *C) {
cond, err := ParseExpression("NetworkErrorRatio() < 0.5")
c.Assert(err, IsNil)
r, err := NewResponseFallback(Response{StatusCode: 200, ContentType: "application/json", Body: []byte("yo")})
c.Assert(err, IsNil)
params := []struct {
Condition threshold.Predicate
Fallback middleware.Middleware
Options Options
}{
{
Condition: cond,
Fallback: nil,
Options: Options{},
},
{
Condition: nil,
Fallback: r,
Options: Options{},
},
{
Condition: cond,
Fallback: r,
Options: Options{
FallbackDuration: -1 * time.Millisecond,
},
},
}
for _, p := range params {
cb, err := New(p.Condition, p.Fallback, p.Options)
c.Assert(err, NotNil)
c.Assert(cb, IsNil)
}
}
type O struct {
stats *metrics.RoundTripMetrics
id int64
noAttempts bool
}
func makeRequest(o O) *request.BaseRequest {
req := request.NewBaseRequest(&http.Request{URL: &url.URL{}}, o.id, nil)
if o.noAttempts {
return req
}
req.SetUserData(cbreakerMetrics, o.stats)
req.Attempts = []request.Attempt{
&request.BaseAttempt{},
}
return req
}
func expr(v string) threshold.Predicate {
e, err := ParseExpression(v)
if err != nil {
panic(err)
}
return e
}
*/
func statsOK() *memmetrics.RTMetrics {
m, err := memmetrics.NewRTMetrics()
if err != nil {
circuitbreaker/predicates.go
-1
View File
@@ -58,7 +58,6 @@ func latencyAtQuantile(quantile float64) toInt {
func networkErrorRatio() toFloat64 {
return func(c *CircuitBreaker) float64 {
c.log.Infof("NetworkErrorRatio() == %v", c.metrics.NetworkErrorRatio())
return c.metrics.NetworkErrorRatio()
}
}
circuitbreaker/predicates_test.go
+17 -28
View File
@@ -1,6 +1,9 @@
package circuitbreaker
import (
"github.com/mailgun/oxy/memmetrics"
"time"
. "gopkg.in/check.v1"
)
@@ -9,81 +12,67 @@ type PredicatesSuite struct {
var _ = Suite(&PredicatesSuite{})
/*
func (s *PredicatesSuite) TestTriggered(c *C) {
func (s *PredicatesSuite) TestTripped(c *C) {
predicates := []struct {
Expression string
Request request.Request
M *memmetrics.RTMetrics
V bool
}{
{
Expression: "NetworkErrorRatio() > 0.5",
Request: makeRequest(O{stats: statsNetErrors(0.6)}),
M: statsNetErrors(0.6),
V: true,
},
{
Expression: "NetworkErrorRatio() < 0.5",
Request: makeRequest(O{stats: statsNetErrors(0.6)}),
M: statsNetErrors(0.6),
V: false,
},
{
Expression: "LatencyAtQuantileMS(50.0) > 50",
Request: makeRequest(O{stats: statsLatencyAtQuantile(50, time.Millisecond*51)}),
M: statsLatencyAtQuantile(50, time.Millisecond*51),
V: true,
},
{
Expression: "LatencyAtQuantileMS(50.0) < 50",
Request: makeRequest(O{stats: statsLatencyAtQuantile(50, time.Millisecond*51)}),
M: statsLatencyAtQuantile(50, time.Millisecond*51),
V: false,
},
{
Expression: "ResponseCodeRatio(500, 600, 0, 600) > 0.5",
Request: makeRequest(O{stats: statsResponseCodes(statusCode{Code: 200, Count: 5}, statusCode{Code: 500, Count: 6})}),
M: statsResponseCodes(statusCode{Code: 200, Count: 5}, statusCode{Code: 500, Count: 6}),
V: true,
},
{
Expression: "ResponseCodeRatio(500, 600, 0, 600) > 0.5",
Request: makeRequest(O{stats: statsResponseCodes(statusCode{Code: 200, Count: 5}, statusCode{Code: 500, Count: 4})}),
M: statsResponseCodes(statusCode{Code: 200, Count: 5}, statusCode{Code: 500, Count: 4}),
V: false,
},
}
for _, t := range predicates {
p, err := ParseExpression(t.Expression)
p, err := parseExpression(t.Expression)
c.Assert(err, IsNil)
c.Assert(p, NotNil)
c.Assert(p(t.Request), Equals, t.V)
c.Assert(p(&CircuitBreaker{metrics: t.M}), Equals, t.V)
}
}
func (s *PredicatesSuite) TestErrors(c *C) {
predicates := []struct {
Expression string
Request request.Request
M *memmetrics.RTMetrics
}{
{
Expression: "LatencyAtQuantileMS(40.0) > 50", // quantile not defined
Request: makeRequest(O{stats: statsNetErrors(0.6)}),
},
{
Expression: "LatencyAtQuantileMS(40.0) > 50", // stats are not defined
Request: makeRequest(O{stats: nil}),
},
{
Expression: "NetworkErrorRatio() > 20.0", // stats are not defined
Request: makeRequest(O{stats: nil}),
},
{
Expression: "NetworkErrorRatio() > 20.0", // no last attempt
Request: makeRequest(O{noAttempts: true}),
M: statsNetErrors(0.6),
},
}
for _, t := range predicates {
p, err := ParseExpression(t.Expression)
p, err := parseExpression(t.Expression)
c.Assert(err, IsNil)
c.Assert(p, NotNil)
c.Assert(p(t.Request), Equals, false)
c.Assert(p(&CircuitBreaker{metrics: t.M}), Equals, false)
}
}
*/