Being busy said the chain was down

Nothing could be read, so it answered that the chain could not be reached. Under
load that is this deployment running out of its own time budget with the chain
answering normally the whole while, and the two send a reader to different
places. A timeout says so and names what to do about it.

Co-authored-by: Hanzo Dev <[email protected]>

api/openapi.yaml

@@ -222,6 +222,14 @@ paths:
           content:
             application/json:
               schema: { $ref: '#/components/schemas/Envelope' }
+        '504':
+          description: |
+            Nothing could be read in time. This deployment is busy rather than
+            the chain being unreachable, which is a different place to look: ask
+            for fewer tokens, or ask again in a moment.
+          content:
+            application/json:
+              schema: { $ref: '#/components/schemas/Envelope' }
 
   /v1/trade/quote:
     post:

pkg/gateway/price_test.go

@@ -241,6 +241,40 @@ func TestAnUnreadableChainSaysSo(t *testing.T) {
 	}
 }
 
+// Running out of time is this deployment being busy, and an endpoint that never
+// answered is the chain. They send a reader to different places, so they are
+// not the same answer.
+func TestBusyIsNotAnOutage(t *testing.T) {
+	slow := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		<-r.Context().Done()
+	}))
+	defer slow.Close()
+
+	s := NewServer(
+		NewRouter(NewRegistry(), true),
+		DefaultServerConfig(),
+		WithChainVenues(&ChainRouters{
+			byChain: map[ChainID]*VenueRouter{ChainIDLux: NewVenueRouter(
+				NewUniswapV3Venue(UniswapV3Config{RPCURL: slow.URL, QuoterAddress: testOther}))},
+			usd: map[ChainID]Numeraires{ChainIDLux: usdLux},
+		}),
+	)
+
+	// The request gives up before the chain does.
+	r := httptest.NewRequest(http.MethodGet, "/v1/trade/price?chainId=96369&token="+luxToken("LETH"), nil)
+	ctx, stop := context.WithCancel(r.Context())
+	stop()
+	w := httptest.NewRecorder()
+	s.mux.ServeHTTP(w, r.WithContext(ctx))
+
+	if w.Code != http.StatusGatewayTimeout {
+		t.Fatalf("a request that ran out of time answered %d: %s", w.Code, w.Body.String())
+	}
+	if strings.Contains(w.Body.String(), "could not be read") {
+		t.Errorf("being busy was reported as the chain being down: %s", w.Body.String())
+	}
+}
+
 // /v1/trade/price, as a screen asks it: several tokens at once, answered in
 // the order asked, with the ones nothing holds coming back without a price.
 func TestPriceAnswersInTheOrderAsked(t *testing.T) {

pkg/gateway/server_catalog.go

@@ -141,10 +141,17 @@ func (s *Server) handlePrice(w http.ResponseWriter, r *http.Request) {
 	}
 	wg.Wait()
 
-	// Every one of them failed to reach the chain. That is a fact about an
-	// endpoint, not about the pools, and answering it with a page of dashes
-	// sends a reader looking for liquidity instead of at their RPC.
+	// Not one of them could be read. Answering that with a page of dashes sends
+	// a reader looking for liquidity instead of at the thing that is wrong —
+	// and which thing that is matters, because the two send them to different
+	// places. Running out of time is this deployment being busy; anything else
+	// is the chain's endpoint.
 	if unreachable == len(tokens) && len(tokens) > 0 {
+		if ctx.Err() != nil {
+			s.writeError(w, http.StatusGatewayTimeout,
+				fmt.Errorf("reading chain %d took longer than %s — ask for fewer tokens, or ask again in a moment", chain, pricesWithin))
+			return
+		}
 		s.writeError(w, http.StatusBadGateway, fmt.Errorf("chain %d could not be read", chain))
 		return
 	}