Files
ai/agent/builtin_tool/web/tools_test.go
zeekayandHanzo Dev bcb27958ec agent: every Responses-API agent gets the web — search, fetch, deep research
The builtin registry held five TIME tools. An agent on /v1/responses asking to
search the web had nothing to call, while /v1/websearch/search, /v1/crawl and
/v1/ask had been live the whole time. The capability existed and no agent could
reach it.

Three tools, registered unconditionally so an agent's tool list does not change
shape with deployment:

  web_search     ranked hits with titles, URLs, snippets
  fetch_url      pages as clean Markdown (this module's own crawl)
  deep_research  a cited report synthesised across sources

EVERY BACKEND IS A SEAM, and that is forced rather than chosen.
agent/builtin_tool/web must stay a LEAF: `object` imports `agent`, `agent`
imports the registry, so a tool reaching into object for Crawl closes the loop —
agent → builtin_tool → web → object → agent — and the module stops compiling.
Search and research are the HOST's anyway (hanzoai/cloud serves them and imports
this module, so it cannot be imported back). Crawl is installed by this module's
bootstrap; the host installs the other two at mount time, beside the balance and
tier readers it already installs.

That shape is also the lesson this repo already paid for: the ai module once
reached its own platform over api.hanzo.ai, which carries a CUSTOMER credential
and answers 401 to a service, and every completion fail-closed at 503 on a
healthy pod. The edge is for customers; in-process is for us.

AN UNAVAILABLE CAPABILITY IS AN ERROR, NEVER AN EMPTY RESULT — the property
worth the most here. A model reads ordinary tool text as a FINDING: hand it "no
results" when the backend was never installed and it concludes the web holds
nothing on the subject, then answers from memory in a confident voice. That is a
fabricated answer caused by a deployment gap, and nothing downstream can tell it
from a real one. A genuinely empty search still returns a normal result, because
that IS a finding.

Bounded on purpose: limit clamped to 10 (a confused call must not exhaust the
agent's context window), 5 URLs per fetch, 20k markdown per page, and a failed
page is NAMED rather than omitted — an agent cannot tell "unreachable" from
"said nothing" if it silently disappears.

fetch_url accepts only absolute http(s). The URLs it is handed come from PAGES
the model read, so following file:// or a custom scheme would read the host's own
disk on behalf of whoever wrote that page. It also accepts a bare string, which
is the shape a model sends most often and is unambiguous.

13 tests, including the refuse-vs-empty property for all three tools, the scheme
filter, the clamps, and that the tools are both REGISTERED and exposed in
GetToolsAsProtocolTools — a tool absent from that list is invisible to agents
however well it is registered.

Co-authored-by: Hanzo Dev <[email protected]>
2026-08-05 01:38:04 -07:00

202 lines
7.3 KiB
Go

// Copyright (c) 2014-present Hanzo AI, Inc.
// Licensed under Apache-2.0. See LICENSE.
package webtools
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"github.com/ThinkInAIXYZ/go-mcp/protocol"
)
func body(t *testing.T, r *protocol.CallToolResult) string {
t.Helper()
if len(r.Content) == 0 {
t.Fatal("result carried no content")
}
tc, ok := r.Content[0].(*protocol.TextContent)
if !ok {
t.Fatalf("content is %T, want *protocol.TextContent", r.Content[0])
}
return tc.Text
}
func reset() { SetSearch(nil); SetResearch(nil); SetFetch(nil) }
// AN UNAVAILABLE CAPABILITY MUST BE AN ERROR, NOT AN EMPTY ANSWER.
//
// This is the single most important property here. A model reads ordinary tool text
// as a FINDING: hand it "no results" when the backend was never installed and it
// concludes the web holds nothing on the subject, then answers from memory in a
// confident voice. That is a fabricated answer caused by a deployment gap, and
// nothing downstream can tell it from a real one. Absent and empty are different
// facts and must arrive as different results.
func TestUnconfiguredCapabilityIsAnErrorNotAnEmptyResult(t *testing.T) {
reset()
ctx := context.Background()
for _, tc := range []struct {
name string
run func() (*protocol.CallToolResult, error)
}{
{"web_search", func() (*protocol.CallToolResult, error) {
return (&SearchTool{}).Execute(ctx, map[string]interface{}{"query": "anything"})
}},
{"deep_research", func() (*protocol.CallToolResult, error) {
return (&ResearchTool{}).Execute(ctx, map[string]interface{}{"question": "anything"})
}},
{"fetch_url", func() (*protocol.CallToolResult, error) {
return (&FetchTool{}).Execute(ctx, map[string]interface{}{"urls": []interface{}{"https://example.com"}})
}},
} {
res, err := tc.run()
if err != nil {
t.Fatalf("%s returned a transport error: %v", tc.name, err)
}
if !res.IsError {
t.Errorf("%s with no backend returned a NON-error result (%q) — a model reads that "+
"as a finding and answers from memory", tc.name, body(t, res))
}
if !strings.Contains(body(t, res), "not available in this deployment") {
t.Errorf("%s does not name the deployment as the cause: %q", tc.name, body(t, res))
}
}
}
// A genuinely empty search IS a finding, and must NOT be an error — otherwise an
// agent retries a query that correctly returned nothing.
func TestGenuinelyEmptySearchIsAFindingNotAnError(t *testing.T) {
reset()
SetSearch(func(context.Context, string, int) ([]SearchResult, error) { return nil, nil })
res, _ := (&SearchTool{}).Execute(context.Background(), map[string]interface{}{"query": "zzz"})
if res.IsError {
t.Error("an empty result set was reported as an error; the search worked and found nothing")
}
}
func TestSearchReturnsResultsAsJSON(t *testing.T) {
reset()
SetSearch(func(_ context.Context, q string, limit int) ([]SearchResult, error) {
return []SearchResult{{Title: "T", URL: "https://e.com", Snippet: "s"}}, nil
})
res, _ := (&SearchTool{}).Execute(context.Background(), map[string]interface{}{"query": "q"})
var got []SearchResult
if err := json.Unmarshal([]byte(body(t, res)), &got); err != nil {
t.Fatalf("results are not JSON a model can parse: %v", err)
}
if len(got) != 1 || got[0].URL != "https://e.com" {
t.Errorf("got %+v", got)
}
}
func TestSearchLimitIsClampedNotTrusted(t *testing.T) {
reset()
var asked int
SetSearch(func(_ context.Context, _ string, limit int) ([]SearchResult, error) {
asked = limit
return nil, nil
})
run := func(v interface{}) int {
asked = 0
(&SearchTool{}).Execute(context.Background(), map[string]interface{}{"query": "q", "limit": v})
return asked
}
// One confused call must not be able to exhaust the agent's context window.
if got := run(float64(9999)); got != maxResults {
t.Errorf("limit 9999 asked for %d, want clamp to %d", got, maxResults)
}
if got := run(float64(-3)); got != defaultResult {
t.Errorf("negative limit asked for %d, want the default %d", got, defaultResult)
}
if got := run("not a number"); got != defaultResult {
t.Errorf("junk limit asked for %d, want the default %d", got, defaultResult)
}
}
// fetch_url must never follow a non-http(s) scheme. The URLs it is handed come from
// PAGES the model read, so following file:// or a custom scheme would read the host's
// own disk on behalf of whoever wrote that page.
func TestFetchRefusesNonHttpSchemes(t *testing.T) {
reset()
var got []string
SetFetch(func(_ context.Context, urls []string) ([]FetchResult, error) {
got = urls
return nil, nil
})
res, _ := (&FetchTool{}).Execute(context.Background(), map[string]interface{}{
"urls": []interface{}{"file:///etc/passwd", "javascript:alert(1)", "ftp://x", "https://ok.com"},
})
if res.IsError {
t.Fatalf("unexpected error: %s", body(t, res))
}
if len(got) != 1 || got[0] != "https://ok.com" {
t.Errorf("fetcher was handed %v — only absolute http(s) may pass", got)
}
}
func TestFetchAcceptsABareStringBecauseModelsSendOne(t *testing.T) {
reset()
var got []string
SetFetch(func(_ context.Context, urls []string) ([]FetchResult, error) { got = urls; return nil, nil })
(&FetchTool{}).Execute(context.Background(), map[string]interface{}{"urls": "https://one.com"})
if len(got) != 1 || got[0] != "https://one.com" {
t.Errorf("a single string url was not accepted: %v", got)
}
}
func TestFetchCapsTheNumberOfURLs(t *testing.T) {
reset()
var got []string
SetFetch(func(_ context.Context, urls []string) ([]FetchResult, error) { got = urls; return nil, nil })
many := make([]interface{}, 0, 20)
for i := 0; i < 20; i++ {
many = append(many, "https://e.com/")
}
(&FetchTool{}).Execute(context.Background(), map[string]interface{}{"urls": many})
if len(got) != maxURLs {
t.Errorf("passed %d urls, want a cap of %d", len(got), maxURLs)
}
}
// A page that failed is NAMED. Silently omitting it looks like an empty page, and an
// agent cannot tell "this source is unreachable" from "this source said nothing".
func TestFetchReportsAFailedPage(t *testing.T) {
reset()
SetFetch(func(context.Context, []string) ([]FetchResult, error) {
return []FetchResult{{URL: "https://gone.com", Success: false}}, nil
})
res, _ := (&FetchTool{}).Execute(context.Background(), map[string]interface{}{
"urls": []interface{}{"https://gone.com"},
})
if !strings.Contains(body(t, res), "could not be fetched") {
t.Errorf("a failed page was not named: %q", body(t, res))
}
}
func TestBackendErrorSurfacesAsAToolError(t *testing.T) {
reset()
SetSearch(func(context.Context, string, int) ([]SearchResult, error) {
return nil, errors.New("upstream is down")
})
res, _ := (&SearchTool{}).Execute(context.Background(), map[string]interface{}{"query": "q"})
if !res.IsError || !strings.Contains(body(t, res), "upstream is down") {
t.Errorf("a backend failure did not surface: isError=%v body=%q", res.IsError, body(t, res))
}
}
func TestEmptyArgumentsAreRefused(t *testing.T) {
reset()
SetSearch(func(context.Context, string, int) ([]SearchResult, error) { return nil, nil })
SetResearch(func(context.Context, string) (string, error) { return "x", nil })
if res, _ := (&SearchTool{}).Execute(context.Background(), map[string]interface{}{"query": " "}); !res.IsError {
t.Error("blank query accepted")
}
if res, _ := (&ResearchTool{}).Execute(context.Background(), map[string]interface{}{}); !res.IsError {
t.Error("missing question accepted")
}
}