hanzo.ai/app/(marketing)/crawl/crawl-client.tsx

'use client'

import { Braces, Database, FileText, Globe, Monitor, Search, Shield, Zap } from 'lucide-react'
import { OSSRevenueBanner } from '@/components/oss/OSSRevenueBanner'
import { ProductFooter } from '@/components/products/ProductFooter'
import { ProductLanding } from '@/components/product/ProductLanding'
import type { ProductLandingProps } from '@/components/product/ProductLanding'
import { useI18n } from '@hanzo/i18n/react'

const docs = 'https://docs.hanzo.ai/docs/openapi/crawl/'

const crawl = (t: ReturnType<typeof useI18n>['t']): ProductLandingProps => ({
  badge: t('AI-Ready Web Crawler'),
  badgeIcon: Globe,
  title: 'Hanzo Crawl',
  tagline: t('Give it a URL, get back Markdown'),
  lede: t('It fetches the page, finds the part a person would actually read, and renders that as Markdown. Navigation, cookie banners and footers do not come with it. A page that needs JavaScript gets a real browser; everything else is a fetch and a parse, which is most of the web and far faster.'),
  stats: {
    hero: true,
    items: [
      { value: 'JS', label: t('Full rendering') },
      { value: t('Parallel'), label: t('High throughput') },
      { value: 'LLM', label: t('Structured extraction') },
      { value: 'MD', label: t('Clean Markdown') },
    ],
  },
  ctas: [
    { label: t('Read the docs'), href: docs },
    { label: 'GitHub', href: 'https://github.com/hanzoai' },
  ],
  features: {
    title: t('Three steps, and a boundary'),
    sub: t('Fetch, extract, render. Then the part that matters when the URL comes from a stranger.'),
    items: [
      {
        icon: FileText,
        title: t('The readable part, not the page'),
        body: t('Deciding which subtree is the article is the hard problem, and it is the one this solves. You get the content and what the page says about itself, without the chrome around it.'),
      },
      {
        icon: Monitor,
        title: t('A browser only when needed'),
        body: t('Most pages are a fetch and a parse. Headless Chromium is a separate service it escalates to for the ones that genuinely need rendering, so you do not pay browser cost for a documentation page.'),
      },
      {
        icon: Braces,
        title: t('Markdown, and the metadata'),
        body: t('Headings stay headings and links keep their targets, resolved against the page they came from. Title, description and canonical URL come back alongside — including the URL after redirects, which is the one worth citing.'),
      },
      {
        icon: Zap,
        title: t('It cannot be pointed at your cluster'),
        body: t('The caller supplies the URL, which makes any crawler a request-forgery primitive. Loopback, link-local and multicast are refused — including the cloud metadata endpoint on 169.254.169.254 that hands out credentials.'),
      },
      {
        icon: Search,
        title: t('Every redirect is checked too'),
        body: t('Blocking the first address is not enough, because a public URL can redirect to a private one. The guard sits on the dialer, so a 302 toward metadata is refused at the hop that matters.'),
      },
      {
        icon: Shield,
        title: t('A tool an agent can call'),
        body: t('It answers on MCP as well as over HTTP, so a model can read a page mid-conversation without you building the plumbing for it.'),
      },
    ],
  },
  steps: {
    title: t('How it works'),
    sub: t('Each step is its own file and can be tested without the other two.'),
    items: [
      {
        n: t('Step {n}', { n: '01' }),
        icon: Globe,
        title: t('Fetch'),
        body: t('One guarded HTTP request. The dialer refuses private and link-local addresses on every hop, so a URL that redirects somewhere it should not reach is stopped there rather than after the fact.'),
      },
      {
        n: t('Step {n}', { n: '02' }),
        icon: FileText,
        title: t('Extract'),
        body: t('The document is reduced to the subtree a reader would call the content, plus the title, description and canonical URL the page claims for itself. If the HTML alone is not enough, this is where a headless browser gets involved.'),
      },
      {
        n: t('Step {n}', { n: '03' }),
        icon: Database,
        title: t('Render'),
        body: t('That subtree becomes Markdown with its structure intact — headings, lists, code and links, with relative URLs resolved against the page. It is the format a model reads best and a person can still diff.'),
      },
    ],
  },
  chips: {
    title: t('Two ways to call it'),
    sub: t('Over HTTP at /v1/crawl when your code is doing the asking, or over MCP when a model is.'),
    items: [t('Static Sites'), t('SPAs'), t('Documentation'), t('E-Commerce'), t('Forums'), 'REST API'],
  },
  plans: {
    title: t('Pricing'),
    sub: t('You are paying for the hosted API and the browsers behind it, not for the code.'),
    footnote:
      t('The crawler is open source, and so is the headless-browser image it escalates to. Run both yourself for nothing. The plans buy you somebody else operating them.'),
    items: [
      {
        name: 'Build',
        price: '$49',
        period: 'mo',
        points: [
          t('{n} pages/month', { n: '50,000' }),
          t('{n} concurrent crawlers', { n: '5' }),
          t('JavaScript rendering'),
          t('Markdown + structured output'),
          t('REST API access'),
        ],
        cta: { label: t('Read the docs'), href: docs },
      },
      {
        name: 'Scale',
        price: '$499',
        period: 'mo',
        pick: true,
        points: [
          t('{n} pages/month', { n: '500,000' }),
          t('{n} concurrent crawlers', { n: '25' }),
          t('Priority JS rendering'),
          t('LLM-based extraction'),
          t('Direct Search + Vector integration'),
        ],
        cta: { label: t('Read the docs'), href: docs },
      },
      {
        name: 'Enterprise',
        price: t('Custom'),
        points: [
          t('Unlimited pages'),
          t('Unlimited concurrent crawlers'),
          t('Dedicated rendering pool'),
          t('Custom extraction schemas'),
          t('Dedicated support + SLA'),
        ],
        cta: { label: t('Contact Sales'), href: '/contact-sales' },
      },
    ],
  },
  finalCta: {
    eyebrow: 'Hanzo Crawl',
    title: t('Send it a URL'),
    sub: (
      <>
        {t('One call, and the page comes back as Markdown.')}
        <br />
        {t('From your code over HTTP, or from a model over MCP.')}
      </>
    ),
    buttons: [
      { label: t('Read the docs'), href: docs },
      { label: t('View on GitHub'), href: 'https://github.com/hanzoai' },
    ],
  },
})

export default function CrawlClient() {
  const { t } = useI18n()
  return (
    <>
      <ProductLanding {...crawl(t)}>
        <OSSRevenueBanner />
      </ProductLanding>
      <ProductFooter slug="crawl" name="Crawl" />
    </>
  )
}