python-sdk/pkg/hanzo-tools-api/scripts/generate_providers.py

#!/usr/bin/env python3
"""Generate provider configurations from APIs.guru with LLM-optimized descriptions from oapis.org."""

import json
import re
import sys
import urllib.request
from concurrent.futures import ThreadPoolExecutor

# Common env var patterns
ENV_VAR_PATTERNS = {
    "stripe": ["STRIPE_API_KEY", "STRIPE_SECRET_KEY"],
    "twilio": ["TWILIO_ACCOUNT_SID", "TWILIO_AUTH_TOKEN"],
    "github": ["GITHUB_TOKEN", "GH_TOKEN"],
    "gitlab": ["GITLAB_TOKEN"],
    "slack": ["SLACK_TOKEN", "SLACK_BOT_TOKEN"],
    "discord": ["DISCORD_TOKEN", "DISCORD_BOT_TOKEN"],
    "shopify": ["SHOPIFY_API_KEY", "SHOPIFY_ACCESS_TOKEN"],
    "hubspot": ["HUBSPOT_API_KEY", "HUBSPOT_ACCESS_TOKEN"],
    "mailchimp": ["MAILCHIMP_API_KEY"],
    "sendgrid": ["SENDGRID_API_KEY"],
    "mailgun": ["MAILGUN_API_KEY"],
    "contentful": ["CONTENTFUL_ACCESS_TOKEN"],
    "airtable": ["AIRTABLE_API_KEY"],
    "notion": ["NOTION_API_KEY", "NOTION_TOKEN"],
    "figma": ["FIGMA_TOKEN", "FIGMA_ACCESS_TOKEN"],
    "asana": ["ASANA_TOKEN", "ASANA_ACCESS_TOKEN"],
    "trello": ["TRELLO_API_KEY", "TRELLO_TOKEN"],
    "jira": ["JIRA_TOKEN", "JIRA_API_TOKEN"],
    "confluence": ["CONFLUENCE_TOKEN"],
    "bitbucket": ["BITBUCKET_TOKEN"],
    "dropbox": ["DROPBOX_ACCESS_TOKEN"],
    "box": ["BOX_ACCESS_TOKEN"],
    "spotify": ["SPOTIFY_CLIENT_ID", "SPOTIFY_CLIENT_SECRET"],
    "youtube": ["YOUTUBE_API_KEY"],
    "twitter": ["TWITTER_API_KEY", "TWITTER_BEARER_TOKEN"],
    "linkedin": ["LINKEDIN_ACCESS_TOKEN"],
    "facebook": ["FACEBOOK_ACCESS_TOKEN"],
    "instagram": ["INSTAGRAM_ACCESS_TOKEN"],
    "pinterest": ["PINTEREST_ACCESS_TOKEN"],
    "reddit": ["REDDIT_CLIENT_ID", "REDDIT_CLIENT_SECRET"],
    "zendesk": ["ZENDESK_API_TOKEN"],
    "freshdesk": ["FRESHDESK_API_KEY"],
    "intercom": ["INTERCOM_ACCESS_TOKEN"],
    "mixpanel": ["MIXPANEL_TOKEN"],
    "segment": ["SEGMENT_WRITE_KEY"],
    "amplitude": ["AMPLITUDE_API_KEY"],
    "plaid": ["PLAID_CLIENT_ID", "PLAID_SECRET"],
    "square": ["SQUARE_ACCESS_TOKEN"],
    "paypal": ["PAYPAL_CLIENT_ID", "PAYPAL_CLIENT_SECRET"],
    "braintree": ["BRAINTREE_MERCHANT_ID"],
    "quickbooks": ["QUICKBOOKS_CLIENT_ID"],
    "xero": ["XERO_CLIENT_ID"],
    "salesforce": ["SALESFORCE_ACCESS_TOKEN"],
    "pipedrive": ["PIPEDRIVE_API_TOKEN"],
    "zoho": ["ZOHO_ACCESS_TOKEN"],
    "monday": ["MONDAY_API_KEY"],
    "clickup": ["CLICKUP_API_KEY"],
    "linear": ["LINEAR_API_KEY"],
    "vercel": ["VERCEL_TOKEN"],
    "netlify": ["NETLIFY_AUTH_TOKEN"],
    "heroku": ["HEROKU_API_KEY"],
    "digitalocean": ["DIGITALOCEAN_TOKEN", "DO_TOKEN"],
    "linode": ["LINODE_TOKEN"],
    "vultr": ["VULTR_API_KEY"],
    "cloudflare": ["CLOUDFLARE_API_TOKEN", "CF_API_TOKEN"],
    "datadog": ["DD_API_KEY", "DATADOG_API_KEY"],
    "newrelic": ["NEW_RELIC_API_KEY"],
    "sentry": ["SENTRY_AUTH_TOKEN"],
    "pagerduty": ["PAGERDUTY_API_KEY"],
    "opsgenie": ["OPSGENIE_API_KEY"],
    "splunk": ["SPLUNK_TOKEN"],
    "elasticsearch": ["ELASTIC_API_KEY"],
    "algolia": ["ALGOLIA_API_KEY"],
    "meilisearch": ["MEILISEARCH_API_KEY"],
    "typesense": ["TYPESENSE_API_KEY"],
    "twitch": ["TWITCH_CLIENT_ID", "TWITCH_CLIENT_SECRET"],
    "ebay": ["EBAY_APP_ID", "EBAY_DEV_ID"],
    "etsy": ["ETSY_API_KEY"],
    "walmart": ["WALMART_CLIENT_ID"],
    "bestbuy": ["BESTBUY_API_KEY"],
    "yelp": ["YELP_API_KEY"],
    "foursquare": ["FOURSQUARE_API_KEY"],
    "tripadvisor": ["TRIPADVISOR_API_KEY"],
    "airbnb": ["AIRBNB_API_KEY"],
    "uber": ["UBER_ACCESS_TOKEN"],
    "lyft": ["LYFT_ACCESS_TOKEN"],
    "doordash": ["DOORDASH_API_KEY"],
    "postmates": ["POSTMATES_API_KEY"],
    "mapbox": ["MAPBOX_ACCESS_TOKEN"],
    "here": ["HERE_API_KEY"],
    "tomtom": ["TOMTOM_API_KEY"],
    "openweathermap": ["OPENWEATHERMAP_API_KEY"],
    "weatherapi": ["WEATHERAPI_KEY"],
    "newsapi": ["NEWSAPI_KEY"],
    "nytimes": ["NYTIMES_API_KEY"],
    "guardian": ["GUARDIAN_API_KEY"],
    "giphy": ["GIPHY_API_KEY"],
    "unsplash": ["UNSPLASH_ACCESS_KEY"],
    "pexels": ["PEXELS_API_KEY"],
    "cloudinary": ["CLOUDINARY_API_KEY"],
    "imgix": ["IMGIX_API_KEY"],
    "uploadcare": ["UPLOADCARE_PUBLIC_KEY"],
    "filestack": ["FILESTACK_API_KEY"],
    "agora": ["AGORA_APP_ID"],
    "vonage": ["VONAGE_API_KEY", "NEXMO_API_KEY"],
    "messagebird": ["MESSAGEBIRD_API_KEY"],
    "bandwidth": ["BANDWIDTH_API_TOKEN"],
    "telnyx": ["TELNYX_API_KEY"],
    "apilayer": ["APILAYER_API_KEY"],
    "currencylayer": ["CURRENCYLAYER_API_KEY"],
    "exchangerate": ["EXCHANGERATE_API_KEY"],
    "coinbase": ["COINBASE_API_KEY"],
    "binance": ["BINANCE_API_KEY"],
    "kraken": ["KRAKEN_API_KEY"],
    "alchemy": ["ALCHEMY_API_KEY"],
    "infura": ["INFURA_PROJECT_ID"],
    "moralis": ["MORALIS_API_KEY"],
    "thegraph": ["THEGRAPH_API_KEY"],
    "openai": ["OPENAI_API_KEY"],
    "anthropic": ["ANTHROPIC_API_KEY"],
    "cohere": ["COHERE_API_KEY"],
    "huggingface": ["HF_TOKEN", "HUGGINGFACE_TOKEN"],
    "replicate": ["REPLICATE_API_TOKEN"],
    "stability": ["STABILITY_API_KEY"],
    "deepl": ["DEEPL_API_KEY"],
    "ably": ["ABLY_API_KEY"],
    "pusher": ["PUSHER_APP_KEY"],
    "pubnub": ["PUBNUB_SUBSCRIBE_KEY"],
    "firebase": ["FIREBASE_API_KEY"],
    "supabase": ["SUPABASE_API_KEY"],
    "mongodb": ["MONGODB_API_KEY"],
    "redis": ["REDIS_API_KEY"],
    "cockroachdb": ["COCKROACHDB_API_KEY"],
    "planetscale": ["PLANETSCALE_TOKEN"],
    "neon": ["NEON_API_KEY"],
    "upstash": ["UPSTASH_API_KEY"],
    "fauna": ["FAUNA_SECRET"],
    "adyen": ["ADYEN_API_KEY"],
    "sportsdata": ["SPORTSDATA_API_KEY"],
    "amadeus": ["AMADEUS_API_KEY", "AMADEUS_API_SECRET"],
    "nexmo": ["NEXMO_API_KEY", "NEXMO_API_SECRET"],
    "mastercard": ["MASTERCARD_API_KEY"],
    "hubapi": ["HUBSPOT_API_KEY"],
    "apideck": ["APIDECK_API_KEY"],
    "codat": ["CODAT_API_KEY"],
    "deutschebahn": ["DEUTSCHEBAHN_API_KEY"],
    "rapidapi": ["RAPIDAPI_KEY"],
    "whapi": ["WHAPI_TOKEN"],
    "vtex": ["VTEX_APP_KEY", "VTEX_APP_TOKEN"],
    "interzoid": ["INTERZOID_API_KEY"],
}

# Skip cloud-specific APIs that need special auth handling
SKIP_PREFIXES = [
    "amazonaws",
    "azure",
    "googleapis",
    "google.",
    "apisetu",
    "parliament",
    "gov.",
    "opto22",
    "windows",
    "o365",
    "microsofthealth",
]

# Popular APIs with oapis.org LLM-friendly descriptions
# These get priority loading and better descriptions
OAPIS_POPULAR = [
    "github",
    "stripe",
    "twilio",
    "slack",
    "notion",
    "discord",
    "shopify",
    "spotify",
    "twitter",
    "openai",
    "anthropic",
]


def get_env_vars(name: str) -> list[str]:
    """Get env vars for a provider name."""
    name_lower = name.lower()
    for key, vars in ENV_VAR_PATTERNS.items():
        if key in name_lower:
            return vars
    # Generate default pattern
    clean = re.sub(r"[^a-z0-9]", "_", name_lower).upper()
    return [f"{clean}_API_KEY", f"{clean}_TOKEN"]


def clean_name(name: str) -> str:
    """Clean API name for use as provider ID."""
    # Remove version suffixes
    name = re.sub(r":\d+.*$", "", name)
    # Replace dots and colons with hyphens
    name = name.replace(".", "-").replace(":", "-")
    # Remove common suffixes
    name = re.sub(r"-com$|-io$|-net$|-org$|-local$", "", name)
    # Clean up
    name = re.sub(r"[^a-z0-9-]", "", name.lower())
    name = re.sub(r"-+", "-", name).strip("-")
    return name


def fetch_oapis_slop(name: str) -> dict | None:
    """Fetch LLM-optimized description from oapis.org/slop/{name}.

    Returns dict with base_url, endpoints_count, description if available.
    """
    try:
        url = f"https://oapis.org/slop/{name}"
        req = urllib.request.Request(url, headers={"User-Agent": "hanzo-tools-api/0.2"})
        with urllib.request.urlopen(req, timeout=10) as response:
            content = response.read().decode()

            # Parse the slop format
            result = {}

            # Extract base URL (usually on a line with https://)
            base_match = re.search(r"Base URL:\s*(https?://[^\s]+)", content, re.IGNORECASE)
            if not base_match:
                base_match = re.search(r"\*\*Base URL\*\*:\s*(https?://[^\s]+)", content, re.IGNORECASE)
            if base_match:
                result["base_url"] = base_match.group(1).rstrip("/")

            # Extract endpoint count
            count_match = re.search(r"(\d+)\s+endpoints?", content, re.IGNORECASE)
            if count_match:
                result["endpoints_count"] = int(count_match.group(1))

            # Extract description (first paragraph after title)
            desc_match = re.search(r"^#[^\n]+\n+([^#\n][^\n]+)", content, re.MULTILINE)
            if desc_match:
                result["description"] = desc_match.group(1).strip()[:200]

            return result if result else None
    except Exception:
        return None


def fetch_apis_guru() -> dict:
    """Fetch API list from APIs.guru."""
    url = "https://api.apis.guru/v2/list.json"
    with urllib.request.urlopen(url, timeout=60) as response:
        return json.loads(response.read().decode())


def fetch_oapis_batch(names: list[str], max_workers: int = 10) -> dict[str, dict]:
    """Fetch oapis.org descriptions for multiple APIs concurrently."""
    results = {}

    def fetch_one(name: str) -> tuple[str, dict | None]:
        return name, fetch_oapis_slop(name)

    with ThreadPoolExecutor(max_workers=max_workers) as executor:
        for name, data in executor.map(lambda n: fetch_one(n), names):
            if data:
                results[name] = data

    return results


def main():
    print("Fetching APIs from APIs.guru...", file=sys.stderr)
    data = fetch_apis_guru()
    print(f"Found {len(data)} APIs", file=sys.stderr)

    # Fetch oapis.org LLM descriptions for popular APIs
    print(f"Fetching LLM descriptions from oapis.org for {len(OAPIS_POPULAR)} popular APIs...", file=sys.stderr)
    oapis_data = fetch_oapis_batch(OAPIS_POPULAR)
    print(f"Got {len(oapis_data)} oapis.org descriptions", file=sys.stderr)

    apis = []
    seen_names = set()

    for name, info in data.items():
        # Skip cloud-specific
        skip = False
        for prefix in SKIP_PREFIXES:
            if name.lower().startswith(prefix.lower()):
                skip = True
                break
        if skip:
            continue

        preferred = info.get("preferred", "")
        if not preferred or preferred not in info.get("versions", {}):
            continue

        version_info = info["versions"][preferred]
        spec_url = version_info.get("swaggerUrl", "")
        if not spec_url:
            continue

        api_info = version_info.get("info", {})
        title = api_info.get("title", name)

        clean = clean_name(name)
        if clean in seen_names or len(clean) < 2:
            continue
        seen_names.add(clean)

        # Get base URL from x-origin
        base_url = ""
        if "x-origin" in api_info:
            x_origin = api_info["x-origin"]
            if isinstance(x_origin, list) and x_origin:
                base_url = x_origin[0].get("url", "")

        # Check for oapis.org enhanced data
        oapis_key = None
        for popular in OAPIS_POPULAR:
            if popular in clean:
                oapis_key = popular
                break

        description = api_info.get("description", "")[:200] if api_info.get("description") else ""
        endpoints_count = None

        if oapis_key and oapis_key in oapis_data:
            oapis_info = oapis_data[oapis_key]
            if oapis_info.get("base_url"):
                base_url = oapis_info["base_url"]
            if oapis_info.get("description"):
                description = oapis_info["description"]
            if oapis_info.get("endpoints_count"):
                endpoints_count = oapis_info["endpoints_count"]

        # Clean strings - remove newlines, escape quotes, limit length
        def clean_str(s: str, max_len: int = 150) -> str:
            if not s:
                return ""
            # Replace various whitespace with single space
            s = " ".join(s.split())
            # Escape backslashes first, then quotes
            s = s.replace("\\", "\\\\").replace('"', '\\"')
            return s[:max_len]

        apis.append(
            {
                "name": clean,
                "title": clean_str(title, 60),
                "description": clean_str(description, 150),
                "spec_url": spec_url,
                "base_url": base_url,
                "env_vars": get_env_vars(clean),
                "endpoints_count": endpoints_count,
            }
        )

    # Sort by name
    apis.sort(key=lambda x: x["name"])

    # Generate Python code
    print('"""Auto-generated provider configurations from APIs.guru + oapis.org.')
    print()
    print(f"Generated {len(apis)} provider configurations.")
    print("Includes LLM-optimized descriptions from oapis.org for popular APIs.")
    print()
    print("Regenerate with: python scripts/generate_providers.py > hanzo_tools/api/apis_guru_providers.py")
    print('"""')
    print()
    print("from typing import Any")
    print()
    print()
    print("# =============================================================================")
    print("# APIs.guru Provider Configurations (with oapis.org enhancements)")
    print("# =============================================================================")
    print()
    print("APIS_GURU_PROVIDERS: dict[str, dict[str, Any]] = {")

    for api in apis:
        env_str = ", ".join(f'"{v}"' for v in api["env_vars"][:2])
        print(f'    "{api["name"]}": {{')
        print(f'        "display_name": "{api["title"]}",')
        if api["description"]:
            print(f'        "description": "{api["description"][:150]}",')
        print(f'        "spec_url": "{api["spec_url"]}",')
        if api["base_url"]:
            safe_url = api["base_url"].replace('"', '\\"')
            print(f'        "base_url": "{safe_url}",')
        print(f'        "env_vars": [{env_str}],')
        if api["endpoints_count"]:
            print(f'        "endpoints_count": {api["endpoints_count"]},')
        print("    },")

    print("}")
    print()
    print()
    print(f"# Total: {len(apis)} providers")

    print(f"\nGenerated {len(apis)} providers", file=sys.stderr)


if __name__ == "__main__":
    main()