#!/usr/bin/env python3 """Generate provider configurations from APIs.guru with LLM-optimized descriptions from oapis.org.""" import asyncio import json import re import sys from pathlib import Path import urllib.request from concurrent.futures import ThreadPoolExecutor # Common env var patterns ENV_VAR_PATTERNS = { 'stripe': ['STRIPE_API_KEY', 'STRIPE_SECRET_KEY'], 'twilio': ['TWILIO_ACCOUNT_SID', 'TWILIO_AUTH_TOKEN'], 'github': ['GITHUB_TOKEN', 'GH_TOKEN'], 'gitlab': ['GITLAB_TOKEN'], 'slack': ['SLACK_TOKEN', 'SLACK_BOT_TOKEN'], 'discord': ['DISCORD_TOKEN', 'DISCORD_BOT_TOKEN'], 'shopify': ['SHOPIFY_API_KEY', 'SHOPIFY_ACCESS_TOKEN'], 'hubspot': ['HUBSPOT_API_KEY', 'HUBSPOT_ACCESS_TOKEN'], 'mailchimp': ['MAILCHIMP_API_KEY'], 'sendgrid': ['SENDGRID_API_KEY'], 'mailgun': ['MAILGUN_API_KEY'], 'contentful': ['CONTENTFUL_ACCESS_TOKEN'], 'airtable': ['AIRTABLE_API_KEY'], 'notion': ['NOTION_API_KEY', 'NOTION_TOKEN'], 'figma': ['FIGMA_TOKEN', 'FIGMA_ACCESS_TOKEN'], 'asana': ['ASANA_TOKEN', 'ASANA_ACCESS_TOKEN'], 'trello': ['TRELLO_API_KEY', 'TRELLO_TOKEN'], 'jira': ['JIRA_TOKEN', 'JIRA_API_TOKEN'], 'confluence': ['CONFLUENCE_TOKEN'], 'bitbucket': ['BITBUCKET_TOKEN'], 'dropbox': ['DROPBOX_ACCESS_TOKEN'], 'box': ['BOX_ACCESS_TOKEN'], 'spotify': ['SPOTIFY_CLIENT_ID', 'SPOTIFY_CLIENT_SECRET'], 'youtube': ['YOUTUBE_API_KEY'], 'twitter': ['TWITTER_API_KEY', 'TWITTER_BEARER_TOKEN'], 'linkedin': ['LINKEDIN_ACCESS_TOKEN'], 'facebook': ['FACEBOOK_ACCESS_TOKEN'], 'instagram': ['INSTAGRAM_ACCESS_TOKEN'], 'pinterest': ['PINTEREST_ACCESS_TOKEN'], 'reddit': ['REDDIT_CLIENT_ID', 'REDDIT_CLIENT_SECRET'], 'zendesk': ['ZENDESK_API_TOKEN'], 'freshdesk': ['FRESHDESK_API_KEY'], 'intercom': ['INTERCOM_ACCESS_TOKEN'], 'mixpanel': ['MIXPANEL_TOKEN'], 'segment': ['SEGMENT_WRITE_KEY'], 'amplitude': ['AMPLITUDE_API_KEY'], 'plaid': ['PLAID_CLIENT_ID', 'PLAID_SECRET'], 'square': ['SQUARE_ACCESS_TOKEN'], 'paypal': ['PAYPAL_CLIENT_ID', 'PAYPAL_CLIENT_SECRET'], 'braintree': ['BRAINTREE_MERCHANT_ID'], 'quickbooks': ['QUICKBOOKS_CLIENT_ID'], 'xero': ['XERO_CLIENT_ID'], 'salesforce': ['SALESFORCE_ACCESS_TOKEN'], 'pipedrive': ['PIPEDRIVE_API_TOKEN'], 'zoho': ['ZOHO_ACCESS_TOKEN'], 'monday': ['MONDAY_API_KEY'], 'clickup': ['CLICKUP_API_KEY'], 'linear': ['LINEAR_API_KEY'], 'vercel': ['VERCEL_TOKEN'], 'netlify': ['NETLIFY_AUTH_TOKEN'], 'heroku': ['HEROKU_API_KEY'], 'digitalocean': ['DIGITALOCEAN_TOKEN', 'DO_TOKEN'], 'linode': ['LINODE_TOKEN'], 'vultr': ['VULTR_API_KEY'], 'cloudflare': ['CLOUDFLARE_API_TOKEN', 'CF_API_TOKEN'], 'datadog': ['DD_API_KEY', 'DATADOG_API_KEY'], 'newrelic': ['NEW_RELIC_API_KEY'], 'sentry': ['SENTRY_AUTH_TOKEN'], 'pagerduty': ['PAGERDUTY_API_KEY'], 'opsgenie': ['OPSGENIE_API_KEY'], 'splunk': ['SPLUNK_TOKEN'], 'elasticsearch': ['ELASTIC_API_KEY'], 'algolia': ['ALGOLIA_API_KEY'], 'meilisearch': ['MEILISEARCH_API_KEY'], 'typesense': ['TYPESENSE_API_KEY'], 'twitch': ['TWITCH_CLIENT_ID', 'TWITCH_CLIENT_SECRET'], 'ebay': ['EBAY_APP_ID', 'EBAY_DEV_ID'], 'etsy': ['ETSY_API_KEY'], 'walmart': ['WALMART_CLIENT_ID'], 'bestbuy': ['BESTBUY_API_KEY'], 'yelp': ['YELP_API_KEY'], 'foursquare': ['FOURSQUARE_API_KEY'], 'tripadvisor': ['TRIPADVISOR_API_KEY'], 'airbnb': ['AIRBNB_API_KEY'], 'uber': ['UBER_ACCESS_TOKEN'], 'lyft': ['LYFT_ACCESS_TOKEN'], 'doordash': ['DOORDASH_API_KEY'], 'postmates': ['POSTMATES_API_KEY'], 'mapbox': ['MAPBOX_ACCESS_TOKEN'], 'here': ['HERE_API_KEY'], 'tomtom': ['TOMTOM_API_KEY'], 'openweathermap': ['OPENWEATHERMAP_API_KEY'], 'weatherapi': ['WEATHERAPI_KEY'], 'newsapi': ['NEWSAPI_KEY'], 'nytimes': ['NYTIMES_API_KEY'], 'guardian': ['GUARDIAN_API_KEY'], 'giphy': ['GIPHY_API_KEY'], 'unsplash': ['UNSPLASH_ACCESS_KEY'], 'pexels': ['PEXELS_API_KEY'], 'cloudinary': ['CLOUDINARY_API_KEY'], 'imgix': ['IMGIX_API_KEY'], 'uploadcare': ['UPLOADCARE_PUBLIC_KEY'], 'filestack': ['FILESTACK_API_KEY'], 'agora': ['AGORA_APP_ID'], 'vonage': ['VONAGE_API_KEY', 'NEXMO_API_KEY'], 'messagebird': ['MESSAGEBIRD_API_KEY'], 'bandwidth': ['BANDWIDTH_API_TOKEN'], 'telnyx': ['TELNYX_API_KEY'], 'apilayer': ['APILAYER_API_KEY'], 'currencylayer': ['CURRENCYLAYER_API_KEY'], 'exchangerate': ['EXCHANGERATE_API_KEY'], 'coinbase': ['COINBASE_API_KEY'], 'binance': ['BINANCE_API_KEY'], 'kraken': ['KRAKEN_API_KEY'], 'alchemy': ['ALCHEMY_API_KEY'], 'infura': ['INFURA_PROJECT_ID'], 'moralis': ['MORALIS_API_KEY'], 'thegraph': ['THEGRAPH_API_KEY'], 'openai': ['OPENAI_API_KEY'], 'anthropic': ['ANTHROPIC_API_KEY'], 'cohere': ['COHERE_API_KEY'], 'huggingface': ['HF_TOKEN', 'HUGGINGFACE_TOKEN'], 'replicate': ['REPLICATE_API_TOKEN'], 'stability': ['STABILITY_API_KEY'], 'deepl': ['DEEPL_API_KEY'], 'ably': ['ABLY_API_KEY'], 'pusher': ['PUSHER_APP_KEY'], 'pubnub': ['PUBNUB_SUBSCRIBE_KEY'], 'firebase': ['FIREBASE_API_KEY'], 'supabase': ['SUPABASE_API_KEY'], 'mongodb': ['MONGODB_API_KEY'], 'redis': ['REDIS_API_KEY'], 'cockroachdb': ['COCKROACHDB_API_KEY'], 'planetscale': ['PLANETSCALE_TOKEN'], 'neon': ['NEON_API_KEY'], 'upstash': ['UPSTASH_API_KEY'], 'fauna': ['FAUNA_SECRET'], 'adyen': ['ADYEN_API_KEY'], 'sportsdata': ['SPORTSDATA_API_KEY'], 'amadeus': ['AMADEUS_API_KEY', 'AMADEUS_API_SECRET'], 'nexmo': ['NEXMO_API_KEY', 'NEXMO_API_SECRET'], 'mastercard': ['MASTERCARD_API_KEY'], 'hubapi': ['HUBSPOT_API_KEY'], 'apideck': ['APIDECK_API_KEY'], 'codat': ['CODAT_API_KEY'], 'deutschebahn': ['DEUTSCHEBAHN_API_KEY'], 'rapidapi': ['RAPIDAPI_KEY'], 'whapi': ['WHAPI_TOKEN'], 'vtex': ['VTEX_APP_KEY', 'VTEX_APP_TOKEN'], 'interzoid': ['INTERZOID_API_KEY'], } # Skip cloud-specific APIs that need special auth handling SKIP_PREFIXES = [ 'amazonaws', 'azure', 'googleapis', 'google.', 'apisetu', 'parliament', 'gov.', 'opto22', 'windows', 'o365', 'microsofthealth', ] # Popular APIs with oapis.org LLM-friendly descriptions # These get priority loading and better descriptions OAPIS_POPULAR = [ 'github', 'stripe', 'twilio', 'slack', 'notion', 'discord', 'shopify', 'spotify', 'twitter', 'openai', 'anthropic', ] def get_env_vars(name: str) -> list[str]: """Get env vars for a provider name.""" name_lower = name.lower() for key, vars in ENV_VAR_PATTERNS.items(): if key in name_lower: return vars # Generate default pattern clean = re.sub(r'[^a-z0-9]', '_', name_lower).upper() return [f'{clean}_API_KEY', f'{clean}_TOKEN'] def clean_name(name: str) -> str: """Clean API name for use as provider ID.""" # Remove version suffixes name = re.sub(r':\d+.*$', '', name) # Replace dots and colons with hyphens name = name.replace('.', '-').replace(':', '-') # Remove common suffixes name = re.sub(r'-com$|-io$|-net$|-org$|-local$', '', name) # Clean up name = re.sub(r'[^a-z0-9-]', '', name.lower()) name = re.sub(r'-+', '-', name).strip('-') return name def fetch_oapis_slop(name: str) -> dict | None: """Fetch LLM-optimized description from oapis.org/slop/{name}. Returns dict with base_url, endpoints_count, description if available. """ try: url = f"https://oapis.org/slop/{name}" req = urllib.request.Request(url, headers={'User-Agent': 'hanzo-tools-api/0.2'}) with urllib.request.urlopen(req, timeout=10) as response: content = response.read().decode() # Parse the slop format result = {} # Extract base URL (usually on a line with https://) base_match = re.search(r'Base URL:\s*(https?://[^\s]+)', content, re.IGNORECASE) if not base_match: base_match = re.search(r'\*\*Base URL\*\*:\s*(https?://[^\s]+)', content, re.IGNORECASE) if base_match: result['base_url'] = base_match.group(1).rstrip('/') # Extract endpoint count count_match = re.search(r'(\d+)\s+endpoints?', content, re.IGNORECASE) if count_match: result['endpoints_count'] = int(count_match.group(1)) # Extract description (first paragraph after title) desc_match = re.search(r'^#[^\n]+\n+([^#\n][^\n]+)', content, re.MULTILINE) if desc_match: result['description'] = desc_match.group(1).strip()[:200] return result if result else None except Exception: return None def fetch_apis_guru() -> dict: """Fetch API list from APIs.guru.""" url = "https://api.apis.guru/v2/list.json" with urllib.request.urlopen(url, timeout=60) as response: return json.loads(response.read().decode()) def fetch_oapis_batch(names: list[str], max_workers: int = 10) -> dict[str, dict]: """Fetch oapis.org descriptions for multiple APIs concurrently.""" results = {} def fetch_one(name: str) -> tuple[str, dict | None]: return name, fetch_oapis_slop(name) with ThreadPoolExecutor(max_workers=max_workers) as executor: for name, data in executor.map(lambda n: fetch_one(n), names): if data: results[name] = data return results def main(): print("Fetching APIs from APIs.guru...", file=sys.stderr) data = fetch_apis_guru() print(f"Found {len(data)} APIs", file=sys.stderr) # Fetch oapis.org LLM descriptions for popular APIs print(f"Fetching LLM descriptions from oapis.org for {len(OAPIS_POPULAR)} popular APIs...", file=sys.stderr) oapis_data = fetch_oapis_batch(OAPIS_POPULAR) print(f"Got {len(oapis_data)} oapis.org descriptions", file=sys.stderr) apis = [] seen_names = set() for name, info in data.items(): # Skip cloud-specific skip = False for prefix in SKIP_PREFIXES: if name.lower().startswith(prefix.lower()): skip = True break if skip: continue preferred = info.get('preferred', '') if not preferred or preferred not in info.get('versions', {}): continue version_info = info['versions'][preferred] spec_url = version_info.get('swaggerUrl', '') if not spec_url: continue api_info = version_info.get('info', {}) title = api_info.get('title', name) clean = clean_name(name) if clean in seen_names or len(clean) < 2: continue seen_names.add(clean) # Get base URL from x-origin base_url = '' if 'x-origin' in api_info: x_origin = api_info['x-origin'] if isinstance(x_origin, list) and x_origin: base_url = x_origin[0].get('url', '') # Check for oapis.org enhanced data oapis_key = None for popular in OAPIS_POPULAR: if popular in clean: oapis_key = popular break description = api_info.get('description', '')[:200] if api_info.get('description') else '' endpoints_count = None if oapis_key and oapis_key in oapis_data: oapis_info = oapis_data[oapis_key] if oapis_info.get('base_url'): base_url = oapis_info['base_url'] if oapis_info.get('description'): description = oapis_info['description'] if oapis_info.get('endpoints_count'): endpoints_count = oapis_info['endpoints_count'] # Clean strings - remove newlines, escape quotes, limit length def clean_str(s: str, max_len: int = 150) -> str: if not s: return "" # Replace various whitespace with single space s = ' '.join(s.split()) # Escape backslashes first, then quotes s = s.replace('\\', '\\\\').replace('"', '\\"') return s[:max_len] apis.append({ 'name': clean, 'title': clean_str(title, 60), 'description': clean_str(description, 150), 'spec_url': spec_url, 'base_url': base_url, 'env_vars': get_env_vars(clean), 'endpoints_count': endpoints_count, }) # Sort by name apis.sort(key=lambda x: x['name']) # Generate Python code print('"""Auto-generated provider configurations from APIs.guru + oapis.org.') print() print(f'Generated {len(apis)} provider configurations.') print('Includes LLM-optimized descriptions from oapis.org for popular APIs.') print() print('Regenerate with: python scripts/generate_providers.py > hanzo_tools/api/apis_guru_providers.py') print('"""') print() print('from typing import Any') print() print() print('# =============================================================================') print('# APIs.guru Provider Configurations (with oapis.org enhancements)') print('# =============================================================================') print() print('APIS_GURU_PROVIDERS: dict[str, dict[str, Any]] = {') for api in apis: env_str = ', '.join(f'"{v}"' for v in api['env_vars'][:2]) print(f' "{api["name"]}": {{') print(f' "display_name": "{api["title"]}",') if api['description']: print(f' "description": "{api["description"][:150]}",') print(f' "spec_url": "{api["spec_url"]}",') if api['base_url']: safe_url = api['base_url'].replace('"', '\\"') print(f' "base_url": "{safe_url}",') print(f' "env_vars": [{env_str}],') if api['endpoints_count']: print(f' "endpoints_count": {api["endpoints_count"]},') print(' },') print('}') print() print() print(f'# Total: {len(apis)} providers') print(f"\nGenerated {len(apis)} providers", file=sys.stderr) if __name__ == '__main__': main()