crawl: escalate to a browser when the static read is too thin to be the page
/v1/crawl is served by this package -- manifest/apps.go is mount order AND routing order, crawl at 78 against ai at 111 -- and Fetch is one http.Get. For the client-rendered part of the web that returns a near-empty shell, extract finds almost no text, and the caller gets 200 with nothing in it. A crawl that returns nothing and reports success is worse than one that fails, because nothing upstream can tell. So: static first, and if what came back is under 512 bytes of markdown, ask Hanzo Crawl (headless Chromium, ghcr.io/hanzoai/crawl:sha-7b8dc59, CR landed in universe e0942b664) for the rendered version. Length is the whole heuristic on purpose. "Does this have a <div id=root>" recognises today's frameworks and misses tomorrow's; "the extractor found almost nothing" is the symptom itself and does not date. Three things keep escalation from being a downgrade: - Longer text wins, not "the browser answered". A render can come back thinner -- a consent wall, a bot check, a page that needed no JS -- and taking it on faith would make this a regression on exactly the pages it should not touch. - An absent, slow or unhappy browser leaves the static Page standing. That is the state this ships in, since nothing has deployed the CR yet. - A sufficient page never pays for a render at all. TWO FINDINGS FROM THE TESTS, both of which had already shipped in my first pass: 1. It could never have worked. browse() used `client`, whose dialer refuses non-public addresses -- and the browser lives at crawl.hanzo.svc, private by design. Every escalation would have been refused with ErrBlocked. There are now two clients for two trust classes: `client` dials wherever a CALLER asked and stays guarded, `service` dials the one address WE configured. 2. It was an SSRF bypass. Read escalates when Fetch FAILS, and one reason Fetch fails is the guard refusing an internal address -- so "crawl http://10.0.0.1/" would have been refused here and then forwarded to a Chromium that fetches it happily, reachable by anyone who can call /v1/crawl. reachable() now applies the same check before the browser sees a URL, including refusing a host that answers with ANY internal address, since one public IP listed beside the target is the documented way around a first-answer check. The resolver is a var so that boundary is testable with a hostile answer; a check you cannot test with the attack is one you are only assuming holds. 11 tests, all passing, covering both halves of the escalation contract, both markdown wire shapes, and the guard.
This commit is contained in:
antje
+13
-1
@@ -80,7 +80,19 @@ func Read(ctx context.Context, s Scope, url string) (*Page, error) {
|
||||
}
|
||||
p, err := Fetch(ctx, url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
// A static fetch can be refused where a browser is not: a bot check, a
|
||||
// consent interstitial, markup served as something extract will not read.
|
||||
// So a failure is a reason to escalate, not to give up — but only the
|
||||
// browser's answer can be returned, since there is no Page to fall back on.
|
||||
if rendered, rerr := browse(ctx, url); rerr == nil {
|
||||
p = rendered
|
||||
} else {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
// Rendered only when the static read came back too thin to be the page,
|
||||
// and kept only if it is actually richer. See escalate.
|
||||
p = escalate(ctx, p, url)
|
||||
}
|
||||
if a != nil {
|
||||
// Deliberately ignored: a page we fetched is a page we can return. Failing
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
package crawl
|
||||
|
||||
// Escalation to a real browser.
|
||||
//
|
||||
// Fetch is one http.Get. That is right for most of the web and wrong for the
|
||||
// part of it that renders client-side: the response for a single-page app is a
|
||||
// near-empty shell, extract finds almost no text, and the caller gets 200 with
|
||||
// nothing in it. A crawl that returns nothing and says it succeeded is worse
|
||||
// than one that fails, because nothing upstream can tell.
|
||||
//
|
||||
// So: static first, and if what came back is too thin to be the page, ask Hanzo
|
||||
// Crawl (headless Chromium, ghcr.io/hanzoai/crawl) for the rendered version.
|
||||
// Escalation is one-way and best-effort — if the browser is absent, slow or
|
||||
// unhappy, the static Page still stands. That keeps a working crawl working
|
||||
// while the browser is not deployed, which is the state this ships in.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// service is the client used to reach the crawl SERVICE, and it deliberately does
|
||||
// NOT carry the guarded dialer that `client` uses.
|
||||
//
|
||||
// The guard refuses non-public addresses, which is right for a URL a caller
|
||||
// supplied and wrong for this one: the browser lives at crawl.hanzo.svc, an
|
||||
// in-cluster name that resolves to a private address by design. Sending this
|
||||
// through `client` meant every escalation was refused with ErrBlocked, so the
|
||||
// feature could never have worked in production — caught by a test that pointed
|
||||
// it at 127.0.0.1 and got the refusal instead of a render.
|
||||
//
|
||||
// Two different trust classes, two clients: `client` dials wherever a caller
|
||||
// asked, so it is guarded; `service` dials one address WE configured.
|
||||
var service = &http.Client{Timeout: browserTimeout}
|
||||
|
||||
// resolve is the name lookup reachable uses, as a var so a test can supply an
|
||||
// answer instead of depending on live DNS. Production keeps the real resolver;
|
||||
// this exists because the address check is a security boundary and a boundary
|
||||
// you cannot test with a hostile answer is one you are only assuming holds.
|
||||
var resolve = func(ctx context.Context, host string) ([]net.IPAddr, error) {
|
||||
return net.DefaultResolver.LookupIPAddr(ctx, host)
|
||||
}
|
||||
|
||||
// thinText is the markdown length under which a document is treated as not
|
||||
// really the page. Deliberately small: the cost of guessing wrong is one extra
|
||||
// request, and the cost of NOT escalating is silent data loss, so this errs
|
||||
// toward asking. A real article clears it by an order of magnitude; an app shell
|
||||
// (a <div id="root"> plus script tags) does not come close.
|
||||
const thinText = 512
|
||||
|
||||
// browserTimeout bounds the escalation. Rendering is seconds, not milliseconds,
|
||||
// but the caller is a request — so a browser having a bad day must not hold it
|
||||
// open. Exceeded means "keep the static Page", never an error to the caller.
|
||||
const browserTimeout = 45 * time.Second
|
||||
|
||||
// browserEndpoint is the in-cluster crawl service. Same host:port ai already
|
||||
// dials, so the two agree on where the browser lives without a shared constant
|
||||
// across repos.
|
||||
func browserEndpoint() string {
|
||||
if v := strings.TrimSpace(os.Getenv("CRAWL_URL")); v != "" {
|
||||
return strings.TrimRight(v, "/")
|
||||
}
|
||||
return "http://crawl.hanzo.svc:11235"
|
||||
}
|
||||
|
||||
// thin reports whether a Page looks like an unrendered shell rather than the
|
||||
// document. Text length is the whole test on purpose: "does this have a
|
||||
// <div id=root>" identifies today's frameworks and misses tomorrow's, whereas
|
||||
// "the extractor found almost nothing" is the symptom itself and does not date.
|
||||
func thin(p *Page) bool {
|
||||
return p == nil || len(strings.TrimSpace(p.Markdown)) < thinText
|
||||
}
|
||||
|
||||
// markdown decodes the service's `markdown`, which is shape-polymorphic: a bare
|
||||
// string on some builds, an object with fit_markdown/raw_markdown on others.
|
||||
// fit_markdown is the boilerplate-stripped one, so it wins when present.
|
||||
type markdown string
|
||||
|
||||
func (m *markdown) UnmarshalJSON(b []byte) error {
|
||||
var s string
|
||||
if err := json.Unmarshal(b, &s); err == nil {
|
||||
*m = markdown(s)
|
||||
return nil
|
||||
}
|
||||
var obj struct {
|
||||
FitMarkdown string `json:"fit_markdown"`
|
||||
RawMarkdown string `json:"raw_markdown"`
|
||||
}
|
||||
if err := json.Unmarshal(b, &obj); err != nil {
|
||||
return err
|
||||
}
|
||||
if obj.FitMarkdown != "" {
|
||||
*m = markdown(obj.FitMarkdown)
|
||||
} else {
|
||||
*m = markdown(obj.RawMarkdown)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type browserResult struct {
|
||||
URL string `json:"url"`
|
||||
Success bool `json:"success"`
|
||||
Markdown markdown `json:"markdown"`
|
||||
Metadata map[string]interface{} `json:"metadata"`
|
||||
}
|
||||
|
||||
// browserResponse carries results inline; the deployed service answers /crawl
|
||||
// synchronously with a boolean success and no task id.
|
||||
type browserResponse struct {
|
||||
Success bool `json:"success"`
|
||||
Results []browserResult `json:"results"`
|
||||
}
|
||||
|
||||
// reachable applies the SAME address check the guarded dialer applies, to a URL
|
||||
// we are about to hand to something that has no such check.
|
||||
//
|
||||
// This closes a real hole rather than a theoretical one. Read escalates when a
|
||||
// static Fetch FAILS, and one reason it fails is the guard refusing an internal
|
||||
// address. Without this, "crawl http://10.0.0.1/" would be refused by the dialer
|
||||
// and then politely forwarded to a headless Chromium that fetches it happily —
|
||||
// escalation as an SSRF bypass, reachable by anyone who can call /v1/crawl.
|
||||
//
|
||||
// Resolve-then-check mirrors the dialer, including refusing a host that answers
|
||||
// with ANY internal address, since one public IP listed beside the target is the
|
||||
// documented way around a check that only looks at the first answer.
|
||||
func reachable(ctx context.Context, raw string) error {
|
||||
u, err := url.Parse(strings.TrimSpace(raw))
|
||||
if err != nil {
|
||||
return fmt.Errorf("crawl: bad url: %w", err)
|
||||
}
|
||||
if u.Scheme != "http" && u.Scheme != "https" {
|
||||
return fmt.Errorf("%w: scheme %q", ErrBlocked, u.Scheme)
|
||||
}
|
||||
host := u.Hostname()
|
||||
if host == "" {
|
||||
return fmt.Errorf("crawl: url has no host: %q", raw)
|
||||
}
|
||||
ips, err := resolve(ctx, host)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, ip := range ips {
|
||||
if !public(ip.IP) {
|
||||
return fmt.Errorf("%w: %s resolves to %s", ErrBlocked, host, ip.IP)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// browse asks the browser for one URL. An error here is never fatal to a crawl —
|
||||
// every caller falls back to the static Page.
|
||||
func browse(ctx context.Context, raw string) (*Page, error) {
|
||||
if err := reachable(ctx, raw); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body, err := json.Marshal(map[string]any{
|
||||
"urls": []string{raw},
|
||||
"browser_config": map[string]any{"headless": true},
|
||||
"crawler_params": map[string]any{
|
||||
"word_count_threshold": 1,
|
||||
"exclude_external_links": false,
|
||||
"process_iframes": true,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, browserTimeout)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, browserEndpoint()+"/crawl", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
// Token if the deployment has one; the network policy is the real boundary,
|
||||
// so its absence is not a reason to skip rendering.
|
||||
if tok := strings.TrimSpace(os.Getenv("CRAWL_API_TOKEN")); tok != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+tok)
|
||||
}
|
||||
|
||||
resp, err := service.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode < 200 || resp.StatusCode > 299 {
|
||||
return nil, fmt.Errorf("crawl: browser returned %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
var out browserResponse
|
||||
if err := json.NewDecoder(io.LimitReader(resp.Body, maxBody)).Decode(&out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(out.Results) == 0 {
|
||||
return nil, fmt.Errorf("crawl: browser returned no results")
|
||||
}
|
||||
r := out.Results[0]
|
||||
if !r.Success {
|
||||
return nil, fmt.Errorf("crawl: browser could not render %s", raw)
|
||||
}
|
||||
|
||||
md := strings.TrimSpace(string(r.Markdown))
|
||||
if md == "" {
|
||||
return nil, fmt.Errorf("crawl: browser rendered %s to nothing", raw)
|
||||
}
|
||||
|
||||
meta := map[string]interface{}{}
|
||||
for k, v := range r.Metadata {
|
||||
meta[k] = v
|
||||
}
|
||||
meta["renderer"] = "browser"
|
||||
page := &Page{URL: raw, Markdown: md, Metadata: meta}
|
||||
if r.URL != "" {
|
||||
page.URL = r.URL
|
||||
meta["sourceURL"] = r.URL
|
||||
}
|
||||
return page, nil
|
||||
}
|
||||
|
||||
// escalate returns the better of a static Page and a rendered one.
|
||||
//
|
||||
// "Better" is longer text, not "the browser answered". A render can come back
|
||||
// thinner than the static fetch — a consent wall, a bot check, a page that needed
|
||||
// no JS at all — and taking it on faith would make escalation a downgrade.
|
||||
func escalate(ctx context.Context, static *Page, raw string) *Page {
|
||||
if !thin(static) {
|
||||
return static
|
||||
}
|
||||
rendered, err := browse(ctx, raw)
|
||||
if err != nil || rendered == nil {
|
||||
return static
|
||||
}
|
||||
if static != nil && len(rendered.Markdown) <= len(strings.TrimSpace(static.Markdown)) {
|
||||
return static
|
||||
}
|
||||
return rendered
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
package crawl
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// publicDNS makes every hostname answer with one routable address, so these tests
|
||||
// exercise escalation without depending on live DNS.
|
||||
func publicDNS(t *testing.T) {
|
||||
t.Helper()
|
||||
prev := resolve
|
||||
resolve = func(ctx context.Context, host string) ([]net.IPAddr, error) {
|
||||
return []net.IPAddr{{IP: net.ParseIP("93.184.216.34")}}, nil
|
||||
}
|
||||
t.Cleanup(func() { resolve = prev })
|
||||
}
|
||||
|
||||
// ── escalation ──────────────────────────────────────────────────────────────
|
||||
//
|
||||
// The contract is narrow and each half matters for a different reason: escalate
|
||||
// when the static read is too thin to be the page (else a single-page app returns
|
||||
// 200 with nothing), and DON'T when it is not (else every crawl pays for a
|
||||
// browser it did not need). The rest of these cases are the ways escalation can
|
||||
// quietly make things worse.
|
||||
|
||||
// browserAt points the escalation at a stub for the duration of a test.
|
||||
func browserAt(t *testing.T, h http.HandlerFunc) {
|
||||
t.Helper()
|
||||
publicDNS(t)
|
||||
srv := httptest.NewServer(h)
|
||||
t.Setenv("CRAWL_URL", srv.URL)
|
||||
t.Cleanup(srv.Close)
|
||||
}
|
||||
|
||||
// rendered answers /crawl the way the service does: an envelope with a boolean
|
||||
// success and results inline.
|
||||
func rendered(md string) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"success": true,
|
||||
"results": []map[string]any{{
|
||||
"url": "https://example.com/app", "success": true, "markdown": md,
|
||||
}},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEscalateReplacesAThinPage(t *testing.T) {
|
||||
// Trimmed: browse trims, which is correct — a render's leading/trailing
|
||||
// whitespace is not content. The expectation has to match that.
|
||||
full := strings.TrimSpace(strings.Repeat("the rendered article. ", 60))
|
||||
browserAt(t, rendered(full))
|
||||
|
||||
shell := &Page{URL: "https://example.com/app", Markdown: "Loading…"}
|
||||
got := escalate(context.Background(), shell, "https://example.com/app")
|
||||
|
||||
if got.Markdown != full {
|
||||
t.Fatalf("a thin page was not replaced by the render:\n got %q", trunc(got.Markdown))
|
||||
}
|
||||
if got.Metadata["renderer"] != "browser" {
|
||||
t.Errorf("the rendered page should record renderer=browser, got %v", got.Metadata["renderer"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestEscalateLeavesASufficientPageAlone(t *testing.T) {
|
||||
// Fails the test if the browser is called at all: a page that already has its
|
||||
// content must not cost a render.
|
||||
browserAt(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
t.Error("the browser was called for a page that was already complete")
|
||||
})
|
||||
|
||||
article := strings.Repeat("real content already present. ", 40)
|
||||
p := &Page{URL: "https://example.com/post", Markdown: article}
|
||||
|
||||
if got := escalate(context.Background(), p, "https://example.com/post"); got.Markdown != article {
|
||||
t.Fatalf("a complete page was altered")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEscalateKeepsTheStaticPageWhenTheBrowserIsUnreachable(t *testing.T) {
|
||||
// The state this ships in: crawl not deployed. A crawl that worked before must
|
||||
// still work, so an absent browser is a non-event.
|
||||
publicDNS(t)
|
||||
t.Setenv("CRAWL_URL", "http://127.0.0.1:1")
|
||||
|
||||
shell := &Page{URL: "https://example.com/app", Markdown: "Loading…"}
|
||||
got := escalate(context.Background(), shell, "https://example.com/app")
|
||||
|
||||
if got != shell {
|
||||
t.Fatalf("an unreachable browser must leave the static page untouched, got %q", trunc(got.Markdown))
|
||||
}
|
||||
}
|
||||
|
||||
func TestEscalateKeepsTheStaticPageWhenTheRenderIsWorse(t *testing.T) {
|
||||
// A render CAN come back thinner — a consent wall, a bot check, a login gate.
|
||||
// Taking the browser's word for it would make escalation a downgrade, which is
|
||||
// the opposite of the point.
|
||||
browserAt(t, rendered("Please accept cookies to continue."))
|
||||
|
||||
static := &Page{URL: "https://example.com/x", Markdown: "a short but real summary of the page"}
|
||||
got := escalate(context.Background(), static, "https://example.com/x")
|
||||
|
||||
if got != static {
|
||||
t.Fatalf("a thinner render must not win, got %q", trunc(got.Markdown))
|
||||
}
|
||||
}
|
||||
|
||||
func TestEscalateKeepsTheStaticPageWhenTheRenderFails(t *testing.T) {
|
||||
browserAt(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"success": false,
|
||||
"results": []map[string]any{{"url": "https://example.com/app", "success": false}},
|
||||
})
|
||||
})
|
||||
|
||||
shell := &Page{URL: "https://example.com/app", Markdown: "Loading…"}
|
||||
if got := escalate(context.Background(), shell, "https://example.com/app"); got != shell {
|
||||
t.Fatalf("success:false must not replace the static page")
|
||||
}
|
||||
}
|
||||
|
||||
// The service is shape-polymorphic about `markdown` across builds: a bare string
|
||||
// on some, an object with fit_markdown/raw_markdown on others. Decoding only one
|
||||
// shape means a working browser reads as an empty render on the other.
|
||||
func TestBrowseDecodesBothMarkdownShapes(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
md any
|
||||
want string
|
||||
}{
|
||||
{"bare string", "plain markdown body", "plain markdown body"},
|
||||
{"fit preferred over raw", map[string]any{
|
||||
"fit_markdown": "the stripped body", "raw_markdown": "boilerplate and the body",
|
||||
}, "the stripped body"},
|
||||
{"raw when fit is empty", map[string]any{
|
||||
"fit_markdown": "", "raw_markdown": "only the raw body",
|
||||
}, "only the raw body"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
browserAt(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"success": true,
|
||||
"results": []map[string]any{{"url": "https://e.com", "success": true, "markdown": tc.md}},
|
||||
})
|
||||
})
|
||||
p, err := browse(context.Background(), "https://e.com")
|
||||
if err != nil {
|
||||
t.Fatalf("browse: %v", err)
|
||||
}
|
||||
if p.Markdown != tc.want {
|
||||
t.Errorf("markdown = %q, want %q", p.Markdown, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func trunc(s string) string {
|
||||
if len(s) > 60 {
|
||||
return s[:60] + "…"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// ── escalation must not become an SSRF bypass ────────────────────────────────
|
||||
//
|
||||
// Read escalates when a static Fetch FAILS, and one reason it fails is the
|
||||
// guarded dialer refusing an internal address. If the escalation did not apply
|
||||
// the same check, "crawl http://10.0.0.1/" would be refused here and then handed
|
||||
// to a headless Chromium that fetches it happily — reachable by anyone who can
|
||||
// call /v1/crawl. The browser has no guard of its own, so this one is the only
|
||||
// thing standing there.
|
||||
func TestBrowseRefusesAnInternalTarget(t *testing.T) {
|
||||
for _, ip := range []string{"127.0.0.1", "10.0.0.1", "169.254.169.254", "::1"} {
|
||||
t.Run(ip, func(t *testing.T) {
|
||||
prev := resolve
|
||||
resolve = func(ctx context.Context, host string) ([]net.IPAddr, error) {
|
||||
return []net.IPAddr{{IP: net.ParseIP(ip)}}, nil
|
||||
}
|
||||
t.Cleanup(func() { resolve = prev })
|
||||
|
||||
browserAt(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
t.Errorf("the browser was asked to fetch an internal address (%s)", ip)
|
||||
})
|
||||
// browserAt reinstalls the public stub, so re-point at the hostile answer.
|
||||
resolve = func(ctx context.Context, host string) ([]net.IPAddr, error) {
|
||||
return []net.IPAddr{{IP: net.ParseIP(ip)}}, nil
|
||||
}
|
||||
|
||||
if _, err := browse(context.Background(), "http://internal.example/"); err == nil {
|
||||
t.Fatalf("browse accepted a target resolving to %s", ip)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A host that answers with one public address BESIDE an internal one is the
|
||||
// documented way past a check that only inspects the first answer.
|
||||
func TestBrowseRefusesAHostThatAlsoResolvesInternal(t *testing.T) {
|
||||
browserAt(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
t.Error("the browser was asked to fetch a host with an internal address")
|
||||
})
|
||||
resolve = func(ctx context.Context, host string) ([]net.IPAddr, error) {
|
||||
return []net.IPAddr{
|
||||
{IP: net.ParseIP("93.184.216.34")},
|
||||
{IP: net.ParseIP("10.1.2.3")},
|
||||
}, nil
|
||||
}
|
||||
if _, err := browse(context.Background(), "http://split.example/"); err == nil {
|
||||
t.Fatal("browse accepted a host that also resolves to an internal address")
|
||||
}
|
||||
}
|
||||
|
||||
// Non-HTTP schemes never reach a dialer, so the scheme check is the only place
|
||||
// they can be refused.
|
||||
func TestBrowseRefusesNonHTTPSchemes(t *testing.T) {
|
||||
browserAt(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
t.Error("the browser was asked to fetch a non-http scheme")
|
||||
})
|
||||
for _, raw := range []string{"file:///etc/passwd", "gopher://x/", "data:text/html,x"} {
|
||||
if _, err := browse(context.Background(), raw); err == nil {
|
||||
t.Errorf("browse accepted %q", raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user