mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
POST /v1/crawl dialled crawl.hanzo.svc.cluster.local:11235 — a name with no
service behind it. Every request came back success:false with empty markdown, so
the endpoint looked unlucky rather than unwired, and nothing ever escalated.
It now fetches through github.com/hanzoai/cloud/apps/crawl, which is the same
code the answer engine's read stage grounds on. One crawl in the binary for the
product's crawl surface, and it is in-process: no service to deploy, no token to
rotate, no batch to poll.
The replacement is also strictly safer than the dial it replaces. apps/crawl
carries a guarded dialer that resolves and checks EVERY hop, redirects included,
and refuses any address that is not public-unicast. /v1/crawl takes a URL from
the caller — that is textbook SSRF reach — and the crawl4ai leg had no such
guard: a caller could have aimed it at the cluster's own metadata service or an
internal admin port and read the response back out as markdown. It cannot now.
Behaviour that did not change: the request body ({url} or {urls}, merged and
deduped), the ten-URL cap, the write-level auth gate, the balance gate, and the
CrawlResult wire shape. A URL that cannot be read still comes back Success:false
rather than failing the batch — nine pages that read are worth more than one
refusal — and results stay in the order asked, keyed by the url ASKED for so a
batching caller can still match them up. Where a page actually came from is in
Metadata["sourceURL"].
What is NOT ripped, and why: object/scraper.go's `browser` engine still calls
CrawlWithCrawl4AI. It needs the per-page internal/external LINK MAP to run its
BFS, and apps/crawl does not expose one — swapping it would silently reduce a
depth-crawl to a single page. It is already guarded by IsCrawl4AIAvailable() and
degrades to the Go scraper, and whether that host resolves in-cluster is not
something this change can verify from outside. Left honest and flagged rather
than half-migrated.
The crawl4ai-shaped projection behind /v1/crawl is deleted along with its tests;
the wire-decode tests that still cover the scraper's leg stay.
Tests: crawl_test.go proves the refusal (a real local server that WOULD have
served content is still refused, as is 169.254.169.254), that a refusal carries
its reason and no content, that order and requested-url keying hold across the
concurrent batch, and that only an empty request is a whole-request error. The
success path is deliberately not faked: apps/crawl has no env switch and no test
hook for non-public addresses, and a hermetic "it fetched a page" test would have
to weaken the exact property being protected.
Co-authored-by: Hanzo Dev <[email protected]>
105 lines
3.4 KiB
Go
105 lines
3.4 KiB
Go
// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package controllers
|
|
|
|
import (
|
|
"encoding/json"
|
|
|
|
"github.com/hanzoai/account"
|
|
|
|
"github.com/hanzoai/ai/object"
|
|
)
|
|
|
|
// maxCrawlURLs bounds a single /v1/crawl request. Each URL is an outbound fetch,
|
|
// so the batch is capped to keep latency and cost bounded.
|
|
const maxCrawlURLs = 10
|
|
|
|
// crawlRequest is the POST /v1/crawl body. A caller may pass a single `url` or a
|
|
// batch of `urls` (or both — they are merged). This is the ONE crawl contract.
|
|
type crawlRequest struct {
|
|
URL string `json:"url,omitempty"`
|
|
Urls []string `json:"urls,omitempty"`
|
|
}
|
|
|
|
// Crawl
|
|
// @Title Crawl
|
|
// @Tag Crawl API
|
|
// @Description crawl one or more URLs through the native crawl and return clean,
|
|
// @Description LLM-ready markdown. The single canonical "crawl a URL, get content
|
|
// @Description back" endpoint.
|
|
// @Param body body controllers.crawlRequest true "Crawl request ({url} or {urls})"
|
|
// @Success 200 {object} controllers.Response "{results: []object.CrawlResult}"
|
|
// @router /crawl [post]
|
|
func (c *ApiController) Crawl() {
|
|
// Write-level auth, fail-closed: crawl performs billable outbound fetches, so
|
|
// it is gated like /v1/scrape (rejects read-only pk-* / widget keys).
|
|
auth := c.requireIndexAuth()
|
|
if auth == nil {
|
|
return
|
|
}
|
|
|
|
var req crawlRequest
|
|
if err := json.Unmarshal(c.Ctx.Input.RequestBody, &req); err != nil {
|
|
c.ResponseError(err.Error())
|
|
return
|
|
}
|
|
|
|
// Merge single `url` and batch `urls` into one deduplicated, non-empty list.
|
|
urls := make([]string, 0, len(req.Urls)+1)
|
|
seen := map[string]struct{}{}
|
|
for _, u := range append([]string{req.URL}, req.Urls...) {
|
|
if u == "" {
|
|
continue
|
|
}
|
|
if _, dup := seen[u]; dup {
|
|
continue
|
|
}
|
|
seen[u] = struct{}{}
|
|
urls = append(urls, u)
|
|
}
|
|
if len(urls) == 0 {
|
|
c.ResponseError("url (or urls) must not be empty")
|
|
return
|
|
}
|
|
if len(urls) > maxCrawlURLs {
|
|
c.ResponseError("too many urls: at most 10 per request")
|
|
return
|
|
}
|
|
|
|
// Balance gate before the expensive crawl, against the billing SUBJECT within
|
|
// the org NAMESPACE — parity with the chat gate, the scrape gate, and the
|
|
// usage debit. (The balance filter also gates this path; this is the same
|
|
// belt-and-suspenders check /v1/scrape performs.)
|
|
if auth.Owner != "" {
|
|
balance, balanceErr := getUserBalance(account.PayerOf(auth.Owner, auth.UserID).Subject(), auth.Owner)
|
|
if balanceErr == nil && balance <= 0 {
|
|
c.ResponseError("insufficient balance for crawl operation. Add funds at https://hanzo.ai/billing")
|
|
return
|
|
}
|
|
}
|
|
|
|
results, err := object.Crawl(urls)
|
|
if err != nil {
|
|
recordSearchUsage(auth, "crawl", "crawl", "error", 0, c.Ctx.Request.RemoteAddr)
|
|
c.ResponseError(err.Error())
|
|
return
|
|
}
|
|
|
|
recordSearchUsage(auth, "crawl", "crawl", "success", len(results), c.Ctx.Request.RemoteAddr)
|
|
|
|
c.Data["json"] = map[string]interface{}{"results": results}
|
|
c.ServeJSON()
|
|
}
|