Files
ai/object/scraper.go
zeekay 99f2e88361 A tenant's URL is fetched by outbound's rule, and never read as a path
Every address a request names now reaches the network through one client,
github.com/hanzoai/outbound (proxy.Public), which judges the address at
the dial, on the IP the socket connects to, and re-dials and re-judges
every redirect.

What it replaces failed three ways. A message href was checked by
util.Fetchable and then fetched by a different client: the name was
resolved twice, so a rebinding answer passed the check, and a redirect
to 169.254.169.254 was followed unchecked. The check itself was
net.IP's predicates, which admit NAT64 64:ff9b::a9fe:a9fe, 6to4,
Teredo, 198.18/15, 240/4 and the TEST-NETs; proxy/public.go's routable
admitted the same. And /v1/ai/rag/embed's url and /v1/ai/rag/ingest's
files[].url had no check at all: a loopback or metadata URL was
fetched, and a scheme-less string was read off this machine's disk.

A tenant document now goes through txt.GetParsedTextFromAddress (http
and https only, over proxy.Public); txt.GetParsedTextFromUrl is for
addresses this system stored. The Go crawler fetches every page and
robots.txt through proxy.Public, and the browser engine is handed only
addresses outbound.Host finds public.
2026-10-18 22:21:42 -07:00

1016 lines
33 KiB
Go

// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package object
import (
"bufio"
"context"
"crypto/sha256"
"fmt"
"net/http"
"net/url"
"strings"
"sync"
"time"
"github.com/hanzoai/outbound"
"golang.org/x/net/html"
"github.com/hanzoai/ai/log"
"github.com/hanzoai/ai/proxy"
)
const (
scraperUserAgent = "HanzoBot/1.0 (+https://hanzo.ai/bot)"
scraperDefaultDepth = 3
scraperDefaultMax = 100
scraperConcurrency = 5
scraperRequestDelay = 200 * time.Millisecond
scraperRequestTimeout = 30 * time.Second
)
// ScrapeRequest is the request body for web scraping operations.
type ScrapeRequest struct {
URL string `json:"url"`
Depth int `json:"depth,omitempty"`
MaxPages int `json:"maxPages,omitempty"`
Selector string `json:"selector,omitempty"`
Tag string `json:"tag,omitempty"`
Store string `json:"store,omitempty"`
Engine string `json:"engine,omitempty"` // "fast" (Go scraper), "browser" (crawl4ai), or "" (auto)
}
// ScrapeResult holds the extracted content from a single page.
type ScrapeResult struct {
URL string `json:"url"`
Title string `json:"title"`
Description string `json:"description"`
Content string `json:"content"`
Headings []Heading `json:"headings"`
Links StringList `json:"links"`
Structured StructuredData `json:"structured"`
// Dropped counts hrefs on the page that address somewhere else on the web but
// could not be parsed into a URL. Each one may be a page the crawl will never
// learn about, and a page a crawl never learns about is one it can delete
// without ever having looked at it — so this is a reason not to mirror.
Dropped int `json:"dropped,omitempty"`
}
// Heading represents a single heading element with its hierarchy level and anchor.
type Heading struct {
Level int `json:"level"`
ID string `json:"id,omitempty"`
Text string `json:"text"`
}
// ContentBlock represents a block of extracted content (paragraph, code, list).
type ContentBlock struct {
Type string `json:"type"` // "paragraph", "code", "list"
Text string `json:"text"`
Section string `json:"section,omitempty"`
}
// StructuredData mirrors the docs framework format for search indexing.
type StructuredData struct {
Headings []Heading `json:"headings"`
Contents []ContentBlock `json:"contents"`
}
// ScrapeStats is the summary returned after a scrape-and-index operation.
type ScrapeStats struct {
PagesScraped int `json:"pagesScraped"`
DocumentsIndexed int `json:"documentsIndexed"`
Engine string `json:"engine"`
Errors StringList `json:"errors,omitempty"`
}
// robotsRules holds parsed robots.txt disallow rules for User-agent: *.
type robotsRules struct {
disallow []string
}
// crawlItem is a BFS queue entry.
type crawlItem struct {
url string
depth int
}
// ScrapePage fetches a single URL and extracts structured content.
func ScrapePage(pageURL string) (*ScrapeResult, error) {
parsed, err := url.Parse(pageURL)
if err != nil {
return nil, fmt.Errorf("invalid URL %q: %w", pageURL, err)
}
if parsed.Scheme == "" {
parsed.Scheme = "https"
}
pageURL = parsed.String()
req, err := http.NewRequest(http.MethodGet, pageURL, nil)
if err != nil {
return nil, fmt.Errorf("failed to build request for %s: %w", pageURL, err)
}
req.Header.Set("User-Agent", scraperUserAgent)
req.Header.Set("Accept", "text/html,application/xhtml+xml")
// The start URL and every link a crawl follows are the tenant's addresses, so
// they are fetched by outbound's client (proxy.Public): judged at the dial, a
// redirect re-judged, never a neighbour on this network.
ctx, cancel := context.WithTimeout(context.Background(), scraperRequestTimeout)
defer cancel()
resp, err := proxy.Public.Do(req.WithContext(ctx))
if err != nil {
return nil, fmt.Errorf("failed to fetch %s: %w", pageURL, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("HTTP %d fetching %s", resp.StatusCode, pageURL)
}
contentType := resp.Header.Get("Content-Type")
if !strings.Contains(contentType, "text/html") && !strings.Contains(contentType, "application/xhtml") {
return nil, fmt.Errorf("non-HTML content type %q for %s", contentType, pageURL)
}
doc, err := html.Parse(resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to parse HTML from %s: %w", pageURL, err)
}
result := &ScrapeResult{URL: pageURL}
extractContent(doc, result, parsed)
return result, nil
}
// frontier is what a crawl learned a site contains, and what it managed to read.
//
// Completeness is the DIFFERENCE between the two. It is never a list of the ways
// a crawl can fall short, because that list is an allowlist: state it as "no
// budget was hit and no fetch failed, therefore the site is whole" and every
// truncation nobody has thought of yet reads as a complete crawl — which is
// licence to delete. Counting what was READ makes the unknown case fail the safe
// way. A page dropped by a path no one enumerated is still a URL sitting in
// discovered with nothing opposite it in read, and that alone withholds the
// mirror.
type frontier struct {
mu sync.Mutex
discovered map[string]bool
read map[string]bool
reason string
}
func newFrontier() *frontier {
return &frontier{discovered: map[string]bool{}, read: map[string]bool{}}
}
// discover records an in-bounds URL the crawl now knows exists. Every link the
// crawl resolves onto its own host goes here, whether or not it is queued —
// including the ones a depth or a budget will keep it from ever reading.
func (f *frontier) discover(u string) {
f.mu.Lock()
f.discovered[u] = true
f.mu.Unlock()
}
// read records a URL the crawl actually fetched and parsed.
func (f *frontier) readPage(u string) {
f.mu.Lock()
f.read[u] = true
f.mu.Unlock()
}
// note names a reason the crawl fell short. The decision does not rest on it —
// that comes from what went unread — but a named reason also stands on its own,
// so a drop that leaves nothing unread still withholds the mirror.
func (f *frontier) note(reason string) {
f.mu.Lock()
if f.reason == "" {
f.reason = reason
}
f.mu.Unlock()
}
// cut is why this crawl is not the whole site, and is empty only when it is.
func (f *frontier) cut() string {
f.mu.Lock()
defer f.mu.Unlock()
unread := 0
for u := range f.discovered {
if !f.read[u] {
unread++
}
}
switch {
case unread == 0 && f.reason == "":
return ""
case unread == 0:
return f.reason
case f.reason == "":
return fmt.Sprintf("%d of %d discovered pages unread", unread, len(f.discovered))
}
return fmt.Sprintf("%s (%d of %d discovered pages unread)", f.reason, unread, len(f.discovered))
}
// CrawlSite performs a crawl starting from req.URL and returns scraped pages.
// Engine selection:
// - "browser": always use crawl4ai (errors if unavailable)
// - "fast": always use the Go HTML scraper
// - "" (auto): try crawl4ai first, fall back to Go scraper if unreachable
//
// The returned engine string indicates which engine was actually used.
//
// cut is why the crawl is not the whole site, and is empty only when every URL
// it discovered on that site was also read. It is what lets a caller tell "this
// page is gone" from "this page was never reached", which are the same absence
// and opposite facts.
func CrawlSite(req *ScrapeRequest) (results []ScrapeResult, crawlErrors []string, engine string, cut string) {
engine = req.Engine
switch engine {
case "browser":
results, crawlErrors, cut = crawlWithBrowserEngine(req)
return results, crawlErrors, "browser", cut
case "fast":
results, crawlErrors, cut = crawlWithGoScraper(req)
return results, crawlErrors, "fast", cut
default:
// Auto mode: try crawl4ai, fall back to Go scraper
if IsCrawl4AIAvailable() {
log.Info("scraper: crawl4ai is available, using browser engine for %s", req.URL)
results, crawlErrors, cut = crawlWithBrowserEngine(req)
return results, crawlErrors, "browser", cut
}
log.Info("scraper: crawl4ai is not available, falling back to Go scraper for %s", req.URL)
results, crawlErrors, cut = crawlWithGoScraper(req)
return results, crawlErrors, "fast", cut
}
}
// crawlWithBrowserEngine uses crawl4ai to crawl URLs with a headless browser.
// It first crawls the start URL, then follows discovered same-domain links up to
// the configured depth and maxPages limits.
func crawlWithBrowserEngine(req *ScrapeRequest) (results []ScrapeResult, crawlErrors []string, cut string) {
maxPages := req.MaxPages
if maxPages <= 0 {
maxPages = scraperDefaultMax
}
maxDepth := req.Depth
if maxDepth <= 0 {
maxDepth = scraperDefaultDepth
}
startURL, err := url.Parse(req.URL)
if err != nil {
return nil, []string{fmt.Sprintf("invalid start URL: %v", err)}, "invalid start URL"
}
if startURL.Scheme == "" {
startURL.Scheme = "https"
}
baseDomain := startURL.Hostname()
// The same robots rules the Go scraper obeys. Both engines must account for a
// blocked page the same way, or which engine ran decides which pages a site
// keeps.
robots := fetchRobotsTxt(startURL.Scheme + "://" + startURL.Host)
visited := make(map[string]bool)
normalizedStart := normalizeURL(startURL.String())
visited[normalizedStart] = true
f := newFrontier()
f.discover(normalizedStart)
// BFS over discovered links, batching URLs to crawl4ai
currentLevel := []string{normalizedStart}
for depth := 0; depth <= maxDepth && len(results) < maxPages && len(currentLevel) > 0; depth++ {
// Cap the batch to remaining page budget
remaining := maxPages - len(results)
batch := currentLevel
if len(batch) > remaining {
batch = batch[:remaining]
f.note("page budget")
}
submitted := make([]string, 0, len(batch))
for _, u := range batch {
if isDisallowed(robots, u) {
log.Info("scraper: robots.txt disallows %s", u)
// Blocked is not gone. It stays discovered and unread.
f.note("robots.txt")
continue
}
// crawl4ai dials what it is handed, and outbound cannot stand in its
// dialler, so each address is judged here before it is handed over
// (outbound.Host). Refused is not gone: it stays discovered and unread.
if err := addressable(u); err != nil {
crawlErrors = append(crawlErrors, fmt.Sprintf("%s: %v", u, err))
f.note("refused address")
continue
}
submitted = append(submitted, u)
}
if len(submitted) == 0 {
break
}
crawl4aiResults, crawlErr := CrawlWithCrawl4AI(submitted)
if crawlErr != nil {
crawlErrors = append(crawlErrors, fmt.Sprintf("crawl4ai batch at depth %d: %v", depth, crawlErr))
f.note("batch failed")
break
}
// Walk what was SUBMITTED, not what came back. A URL the service drops —
// an internal cap, a swallowed exception, a redirect collapsed onto
// another result — returns nothing at all: no result, no Success:false, no
// error. Iterating the response would never see it, and it is already
// marked visited, so nothing ever retries it.
returned := make(map[string]Crawl4AIResult, len(crawl4aiResults))
for _, cr := range crawl4aiResults {
returned[normalizeURL(cr.URL)] = cr
}
var nextLevel []string
for _, u := range submitted {
cr, ok := returned[u]
if !ok {
f.note("no result returned")
continue
}
if !cr.Success {
crawlErrors = append(crawlErrors, fmt.Sprintf("%s: crawl4ai reported failure", cr.URL))
continue
}
sr := Crawl4AIResultToScrapeResult(cr)
results = append(results, sr)
f.readPage(u)
if sr.Dropped > 0 {
f.note("unparsable link")
}
if len(results) >= maxPages {
f.note("page budget")
break
}
// Collect same-domain links for the next BFS level. Everything on this
// host is discovered here whether or not it is queued, so a link the
// depth limit stops us following still counts against completeness.
for _, link := range sr.Links {
linkParsed, linkErr := url.Parse(link)
if linkErr != nil {
f.note("unparsable link")
continue
}
if linkParsed.Hostname() != baseDomain {
continue
}
normalized := normalizeURL(link)
f.discover(normalized)
if depth >= maxDepth || visited[normalized] {
continue
}
visited[normalized] = true
nextLevel = append(nextLevel, normalized)
}
}
currentLevel = nextLevel
}
return results, crawlErrors, f.cut()
}
// crawlWithGoScraper performs a BFS crawl using the built-in Go HTML scraper.
func crawlWithGoScraper(req *ScrapeRequest) (results []ScrapeResult, crawlErrors []string, cut string) {
maxPages := req.MaxPages
if maxPages <= 0 {
maxPages = scraperDefaultMax
}
maxDepth := req.Depth
if maxDepth <= 0 {
maxDepth = scraperDefaultDepth
}
startURL, err := url.Parse(req.URL)
if err != nil {
return nil, []string{fmt.Sprintf("invalid start URL: %v", err)}, "invalid start URL"
}
if startURL.Scheme == "" {
startURL.Scheme = "https"
}
baseDomain := startURL.Hostname()
robots := fetchRobotsTxt(startURL.Scheme + "://" + startURL.Host)
visited := &sync.Map{}
var resultsMu sync.Mutex
var errorsMu sync.Mutex
f := newFrontier()
sem := make(chan struct{}, scraperConcurrency)
queue := make(chan crawlItem, maxPages*2)
// pending counts items that are QUEUED BUT NOT YET FINISHED. A page adds its
// children before releasing itself, so the count reaches zero only when the
// frontier is empty — and that is the only moment the queue may be closed.
//
// It replaces a wait that counted running pages instead of outstanding ones.
// That count starts at zero, so the close fired before the first page had even
// been picked up, the loop below ended on the closed queue, and the crawl
// returned while its own fetches were still in flight: no pages, no errors, no
// sign anything had gone wrong. Every "fast" crawl indexed nothing.
var pending sync.WaitGroup
normalizedStart := normalizeURL(startURL.String())
visited.Store(normalizedStart, true)
f.discover(normalizedStart)
pending.Add(1)
queue <- crawlItem{url: normalizedStart, depth: 0}
go func() {
pending.Wait()
close(queue)
}()
var pageCount int
var pageCountMu sync.Mutex
// Every path that takes an item off the queue releases exactly one count,
// including the two that decline to crawl it — miss one and the close never
// comes.
for item := range queue {
pageCountMu.Lock()
budget := pageCount >= maxPages
pageCountMu.Unlock()
if budget {
f.note("page budget")
pending.Done()
continue
}
if item.depth > maxDepth {
f.note("depth limit")
pending.Done()
continue
}
sem <- struct{}{}
go func(ci crawlItem) {
defer pending.Done()
defer func() { <-sem }()
pageCountMu.Lock()
if pageCount >= maxPages {
pageCountMu.Unlock()
f.note("page budget")
return
}
pageCount++
pageCountMu.Unlock()
time.Sleep(scraperRequestDelay)
if isDisallowed(robots, ci.url) {
log.Info("scraper: robots.txt disallows %s", ci.url)
// A page we are not allowed to read is not a page that is gone.
// One robots rule arriving must never empty what it now covers.
f.note("robots.txt")
return
}
result, scrapeErr := ScrapePage(ci.url)
if scrapeErr != nil {
errorsMu.Lock()
crawlErrors = append(crawlErrors, fmt.Sprintf("%s: %v", ci.url, scrapeErr))
errorsMu.Unlock()
return
}
resultsMu.Lock()
results = append(results, *result)
resultsMu.Unlock()
f.readPage(ci.url)
if result.Dropped > 0 {
f.note("unparsable link")
}
for _, link := range result.Links {
linkParsed, linkErr := url.Parse(link)
if linkErr != nil {
f.note("unparsable link")
continue
}
if linkParsed.Hostname() != baseDomain {
continue
}
normalized := normalizeURL(link)
// Discovered whether or not it is queued: a link the depth limit
// stops us following is still a page we know exists and did not read.
f.discover(normalized)
if ci.depth >= maxDepth {
continue
}
if _, loaded := visited.LoadOrStore(normalized, true); loaded {
continue
}
// Claimed before the send, released if the send is declined: the
// count may not drop to zero while this page still has children to
// hand over.
pending.Add(1)
select {
case queue <- crawlItem{url: normalized, depth: ci.depth + 1}:
default:
pending.Done()
f.note("queue full")
}
}
}(item)
}
// The loop ends when the queue closes, which is when pending reached zero,
// which is after the last page finished. Everything appended is visible here.
return results, crawlErrors, f.cut()
}
// ScrapeAndIndex crawls a site and indexes the results into the owner's search index.
// The owner parameter determines tenant isolation -- each org gets its own index namespace.
// If Hanzo Storage is configured, crawl results are archived asynchronously for persistence.
func ScrapeAndIndex(owner string, req *ScrapeRequest, lang string) (*ScrapeStats, error) {
if req.URL == "" {
return nil, fmt.Errorf("url must not be empty")
}
results, crawlErrors, engine, cut := CrawlSite(req)
stats := &ScrapeStats{
PagesScraped: len(results),
Engine: engine,
Errors: crawlErrors,
}
if len(results) == 0 {
return stats, nil
}
// Archive crawl results to Hanzo Storage (fire-and-forget)
if IsCrawlStorageConfigured() {
jobID := hashID(fmt.Sprintf("%s-%s-%d", owner, req.URL, time.Now().UnixNano()))
archiveCrawlResultAsync(owner, jobID, results, nil)
}
tag := req.Tag
if tag == "" {
parsed, err := url.Parse(req.URL)
if err == nil {
tag = parsed.Hostname()
}
}
store, err := ResolveStore(owner, req.Store, "docs-hanzo-ai")
if err != nil {
return stats, err
}
docs := scrapeResultsToDocIndex(results, tag)
indexReq := &DocIndexRequest{Documents: docs}
// A crawl that read every page it discovered IS the site's current state, so
// the index can mirror it and a page that was taken down stops being cited. A
// crawl that left anything it found unread saw PART of the site, and a page
// missing from part of a site is not a page that was removed from it — so that
// crawl only adds, exactly as every crawl did before. The tag and the root are
// what bound the mirror; without both there is nothing to bound it to.
//
// A consequence worth knowing before someone reports it as a fault: a site
// with more pages than MaxPages (100 by default) always exhausts the budget,
// always leaves pages unread, and therefore never mirrors. Deletions on a
// large site do not begin until the budget covers it. That is the guard doing
// its job — a crawl that saw 100 of 4000 pages knows nothing about the other
// 3900 — not deletion being broken.
root := crawlRoot(req.URL)
switch {
case tag == "" || root == "":
log.Info("scraper: %s indexed additively, nothing to bound a mirror with", req.URL)
case cut != "":
log.Info("scraper: %s indexed additively, crawl cut short by %s", req.URL, cut)
case len(crawlErrors) > 0:
log.Info("scraper: %s indexed additively, %d page(s) failed to fetch", req.URL, len(crawlErrors))
default:
indexReq.Mirror = &Mirror{Tag: tag, Root: root}
}
count, err := IndexDocuments(owner, store, indexReq, lang)
if err != nil {
return stats, fmt.Errorf("indexing failed after scraping %d pages: %w", len(results), err)
}
stats.DocumentsIndexed = count
return stats, nil
}
// crawlRoot is the subtree a crawl of rawURL is authoritative over: the start
// URL, spelled the way the pages it fetched are spelled. It defaults the scheme
// and drops the trailing slash exactly as the crawler and ScrapePage do, because
// a root that does not match the stored URLs bounds a mirror to nothing — which
// costs a prune, and is the right way to be wrong.
func crawlRoot(rawURL string) string {
parsed, err := url.Parse(rawURL)
if err != nil {
return ""
}
if parsed.Scheme == "" {
parsed.Scheme = "https"
}
return normalizeURL(parsed.String())
}
// scrapeResultsToDocIndex converts scrape results to the DocIndex format for search indexing.
func scrapeResultsToDocIndex(results []ScrapeResult, tag string) []DocIndex {
var docs []DocIndex
for _, result := range results {
pageID := hashID(result.URL)
// Create a document for the page itself
pageDoc := DocIndex{
ID: pageID,
PageID: pageID,
Title: result.Title,
URL: result.URL,
Content: truncateContent(result.Content, 10000),
Tag: tag,
Breadcrumbs: []string{result.Title},
}
docs = append(docs, pageDoc)
// Create a document for each heading + its following content
currentSection := ""
currentSectionID := ""
var sectionContent strings.Builder
for _, block := range result.Structured.Contents {
if block.Section != currentSection && currentSection != "" {
sectionDoc := DocIndex{
ID: hashID(result.URL + "#" + currentSectionID),
PageID: pageID,
Title: result.Title,
URL: sectionURL(result.URL, currentSectionID),
Content: truncateContent(sectionContent.String(), 5000),
Section: currentSection,
SectionID: currentSectionID,
Tag: tag,
Breadcrumbs: []string{result.Title, currentSection},
}
docs = append(docs, sectionDoc)
sectionContent.Reset()
}
if block.Section != "" {
currentSection = block.Section
currentSectionID = slugify(block.Section)
}
if sectionContent.Len() > 0 {
sectionContent.WriteString("\n")
}
sectionContent.WriteString(block.Text)
}
// Flush the final section
if currentSection != "" && sectionContent.Len() > 0 {
sectionDoc := DocIndex{
ID: hashID(result.URL + "#" + currentSectionID),
PageID: pageID,
Title: result.Title,
URL: sectionURL(result.URL, currentSectionID),
Content: truncateContent(sectionContent.String(), 5000),
Section: currentSection,
SectionID: currentSectionID,
Tag: tag,
Breadcrumbs: []string{result.Title, currentSection},
}
docs = append(docs, sectionDoc)
}
}
return docs
}
// extractContent walks the HTML tree and populates the ScrapeResult.
func extractContent(doc *html.Node, result *ScrapeResult, baseURL *url.URL) {
var contentBuilder strings.Builder
var currentSection string
var walk func(*html.Node)
walk = func(n *html.Node) {
if n.Type == html.ElementNode {
tag := strings.ToLower(n.Data)
// Skip elements that are not content
if isSkippedTag(tag) {
return
}
switch {
case tag == "title":
result.Title = getTextContent(n)
return
case tag == "meta":
name := getAttr(n, "name")
if strings.EqualFold(name, "description") {
result.Description = getAttr(n, "content")
}
return
case isHeadingTag(tag):
level := int(tag[1] - '0')
id := getAttr(n, "id")
text := getTextContent(n)
heading := Heading{Level: level, ID: id, Text: text}
result.Headings = append(result.Headings, heading)
result.Structured.Headings = append(result.Structured.Headings, heading)
currentSection = text
case tag == "p":
text := strings.TrimSpace(getTextContent(n))
if text != "" {
contentBuilder.WriteString(text)
contentBuilder.WriteString("\n")
result.Structured.Contents = append(result.Structured.Contents, ContentBlock{
Type: "paragraph",
Text: text,
Section: currentSection,
})
}
return
case tag == "pre" || tag == "code":
text := strings.TrimSpace(getTextContent(n))
if text != "" {
contentBuilder.WriteString(text)
contentBuilder.WriteString("\n")
result.Structured.Contents = append(result.Structured.Contents, ContentBlock{
Type: "code",
Text: text,
Section: currentSection,
})
}
return
case tag == "ul" || tag == "ol":
text := strings.TrimSpace(extractListItems(n))
if text != "" {
contentBuilder.WriteString(text)
contentBuilder.WriteString("\n")
result.Structured.Contents = append(result.Structured.Contents, ContentBlock{
Type: "list",
Text: text,
Section: currentSection,
})
}
return
}
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
walk(c)
}
}
walk(doc)
result.Content = strings.TrimSpace(contentBuilder.String())
// Links come from their own walk of the whole tree, including the containers
// the walk above stops at and the regions it skips — and they resolve against
// <base href> when the document sets one, the way a browser does.
linkBase := baseURL
if b := baseHref(doc, baseURL); b != nil {
linkBase = b
}
collectLinks(doc, linkBase, result)
}
// getTextContent recursively extracts text from a node and its children.
func getTextContent(n *html.Node) string {
if n.Type == html.TextNode {
return n.Data
}
var sb strings.Builder
for c := n.FirstChild; c != nil; c = c.NextSibling {
if c.Type == html.ElementNode && isSkippedTag(strings.ToLower(c.Data)) {
continue
}
sb.WriteString(getTextContent(c))
}
return sb.String()
}
// getAttr returns the value of the named attribute, or empty string.
func getAttr(n *html.Node, key string) string {
for _, a := range n.Attr {
if a.Key == key {
return a.Val
}
}
return ""
}
// isHeadingTag returns true for h1-h6.
func isHeadingTag(tag string) bool {
return len(tag) == 2 && tag[0] == 'h' && tag[1] >= '1' && tag[1] <= '6'
}
// isSkippedTag returns true for tags whose content should be stripped.
func isSkippedTag(tag string) bool {
switch tag {
case "script", "style", "nav", "footer", "header", "aside", "noscript", "iframe", "svg":
return true
}
return false
}
// extractListItems extracts text from li elements within a list.
func extractListItems(n *html.Node) string {
var sb strings.Builder
for c := n.FirstChild; c != nil; c = c.NextSibling {
if c.Type == html.ElementNode && strings.ToLower(c.Data) == "li" {
text := strings.TrimSpace(getTextContent(c))
if text != "" {
sb.WriteString("- ")
sb.WriteString(text)
sb.WriteString("\n")
}
}
}
return sb.String()
}
// pageHref reports whether an href addresses a page at all. A fragment, a
// mailto: or a javascript: goes nowhere the crawl could follow, so failing to
// resolve one costs it nothing — which is what separates those from an href that
// is simply malformed.
func pageHref(href string) bool {
href = strings.TrimSpace(href)
return href != "" &&
!strings.HasPrefix(href, "#") &&
!strings.HasPrefix(href, "javascript:") &&
!strings.HasPrefix(href, "mailto:")
}
// baseHref is the document's <base href> resolved against the page URL, or nil
// when the document sets none.
//
// A browser resolves every relative href on the page against it. A crawl that
// ignores it walks to URLs the site does not serve and never reaches the ones it
// does — and when the misresolved URL happens to answer 200, the crawl reads a
// page it was never pointed at, leaves the real one undiscovered, and every
// signal still reads complete. Doxygen, Javadoc and Jekyll with a baseurl all
// emit one, so this is ordinary docs markup, not an edge case.
//
// Only the first <base href> counts, which is what the HTML spec says too.
func baseHref(n *html.Node, pageURL *url.URL) *url.URL {
var found *url.URL
var walk func(*html.Node)
walk = func(n *html.Node) {
if found != nil {
return
}
if n.Type == html.ElementNode && strings.ToLower(n.Data) == "base" {
if href := strings.TrimSpace(getAttr(n, "href")); href != "" {
if parsed, err := url.Parse(href); err == nil {
found = pageURL.ResolveReference(parsed)
return
}
}
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
walk(c)
}
}
walk(n)
return found
}
// collectLinks walks the WHOLE document for hrefs and resolves them against the
// page they were found on.
//
// It is a pass of its own, and that is the point. Text extraction stops
// descending the moment it has a paragraph's or a list's text, and skips nav,
// header, footer and aside outright — right for prose, wrong for links, because
// a docs site keeps its links in precisely those places: the sidebar tree, the
// next/prev footer, the reference in the middle of a sentence. Riding discovery
// on that walk meant a page linked only from a <ul> was never discovered, never
// read, and then deleted as missing while it was still being served. Discovery
// and extraction answer different questions and no longer share a walk.
func collectLinks(n *html.Node, baseURL *url.URL, result *ScrapeResult) {
if n.Type == html.ElementNode && strings.ToLower(n.Data) == "a" {
if href := getAttr(n, "href"); href != "" {
if resolved := resolveHref(href, baseURL); resolved != "" {
result.Links = append(result.Links, resolved)
} else if pageHref(href) {
result.Dropped++
}
}
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
collectLinks(c, baseURL, result)
}
}
// resolveHref resolves a relative or absolute href against the base URL.
func resolveHref(href string, baseURL *url.URL) string {
href = strings.TrimSpace(href)
if !pageHref(href) {
return ""
}
parsed, err := url.Parse(href)
if err != nil {
return ""
}
resolved := baseURL.ResolveReference(parsed)
// Strip fragment
resolved.Fragment = ""
return resolved.String()
}
// normalizeURL strips trailing slashes and fragments for deduplication.
func normalizeURL(rawURL string) string {
parsed, err := url.Parse(rawURL)
if err != nil {
return rawURL
}
parsed.Fragment = ""
result := parsed.String()
result = strings.TrimRight(result, "/")
return result
}
// addressable judges an address a crawl is about to hand to the browser engine:
// http or https, and a host outbound.Host finds on the public internet.
func addressable(raw string) error {
u, err := url.Parse(raw)
if err != nil {
return err
}
if u.Scheme != "http" && u.Scheme != "https" {
return fmt.Errorf("%w: scheme %q", outbound.ErrRefused, u.Scheme)
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
return judgeHost(ctx, u.Hostname())
}
// judgeHost is outbound.Host, a variable so a crawl test can hand the browser
// engine its loopback site the way it swaps proxy.Public; nothing else assigns it.
var judgeHost = outbound.Host
// fetchRobotsTxt fetches and parses robots.txt for the given origin.
func fetchRobotsTxt(origin string) *robotsRules {
rules := &robotsRules{}
robotsURL := strings.TrimRight(origin, "/") + "/robots.txt"
req, err := http.NewRequest(http.MethodGet, robotsURL, nil)
if err != nil {
return rules
}
req.Header.Set("User-Agent", scraperUserAgent)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
resp, err := proxy.Public.Do(req.WithContext(ctx))
if err != nil {
return rules
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return rules
}
scanner := bufio.NewScanner(resp.Body)
inWildcardAgent := false
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if strings.HasPrefix(line, "#") || line == "" {
continue
}
lower := strings.ToLower(line)
if after, ok := strings.CutPrefix(lower, "user-agent:"); ok {
agent := strings.TrimSpace(after)
inWildcardAgent = (agent == "*")
continue
}
if inWildcardAgent && strings.HasPrefix(lower, "disallow:") {
path := strings.TrimSpace(strings.TrimPrefix(line, strings.SplitN(line, ":", 2)[0]+":"))
if path != "" {
rules.disallow = append(rules.disallow, path)
}
}
}
return rules
}
// isDisallowed checks whether a URL path is disallowed by the robots.txt rules.
func isDisallowed(rules *robotsRules, rawURL string) bool {
if rules == nil || len(rules.disallow) == 0 {
return false
}
parsed, err := url.Parse(rawURL)
if err != nil {
return false
}
path := parsed.Path
for _, disallowed := range rules.disallow {
if strings.HasPrefix(path, disallowed) {
return true
}
}
return false
}
// hashID produces a deterministic short ID from a string.
func hashID(s string) string {
h := sha256.Sum256([]byte(s))
return fmt.Sprintf("%x", h[:12])
}
// slugify converts a heading text to a URL-safe slug.
func slugify(s string) string {
s = strings.ToLower(s)
var sb strings.Builder
for _, r := range s {
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
sb.WriteRune(r)
} else if r == ' ' || r == '-' || r == '_' {
sb.WriteRune('-')
}
}
result := sb.String()
result = strings.Trim(result, "-")
// Collapse multiple dashes
for strings.Contains(result, "--") {
result = strings.ReplaceAll(result, "--", "-")
}
return result
}
// sectionURL appends a fragment to a URL if the sectionID is non-empty.
func sectionURL(pageURL, sectionID string) string {
if sectionID == "" {
return pageURL
}
return pageURL + "#" + sectionID
}
// truncateContent limits content to maxLen characters.
func truncateContent(s string, maxLen int) string {
if len(s) <= maxLen {
return s
}
return s[:maxLen]
}