mirror of
https://github.com/hanzoai/python-sdk.git
synced 2026-10-26 11:49:36 +00:00
style: auto-fix ruff lint and format for Python 3.12
Run ruff check --fix and ruff format across the repo to fix ~495 auto-fixable lint errors (mostly I001 import sorting) and reformat 225 files to match the py312 target. Co-authored-by: Hanzo Dev <[email protected]>
This commit is contained in:
@@ -1,170 +1,185 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate provider configurations from APIs.guru with LLM-optimized descriptions from oapis.org."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import urllib.request
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
# Common env var patterns
|
||||
ENV_VAR_PATTERNS = {
|
||||
'stripe': ['STRIPE_API_KEY', 'STRIPE_SECRET_KEY'],
|
||||
'twilio': ['TWILIO_ACCOUNT_SID', 'TWILIO_AUTH_TOKEN'],
|
||||
'github': ['GITHUB_TOKEN', 'GH_TOKEN'],
|
||||
'gitlab': ['GITLAB_TOKEN'],
|
||||
'slack': ['SLACK_TOKEN', 'SLACK_BOT_TOKEN'],
|
||||
'discord': ['DISCORD_TOKEN', 'DISCORD_BOT_TOKEN'],
|
||||
'shopify': ['SHOPIFY_API_KEY', 'SHOPIFY_ACCESS_TOKEN'],
|
||||
'hubspot': ['HUBSPOT_API_KEY', 'HUBSPOT_ACCESS_TOKEN'],
|
||||
'mailchimp': ['MAILCHIMP_API_KEY'],
|
||||
'sendgrid': ['SENDGRID_API_KEY'],
|
||||
'mailgun': ['MAILGUN_API_KEY'],
|
||||
'contentful': ['CONTENTFUL_ACCESS_TOKEN'],
|
||||
'airtable': ['AIRTABLE_API_KEY'],
|
||||
'notion': ['NOTION_API_KEY', 'NOTION_TOKEN'],
|
||||
'figma': ['FIGMA_TOKEN', 'FIGMA_ACCESS_TOKEN'],
|
||||
'asana': ['ASANA_TOKEN', 'ASANA_ACCESS_TOKEN'],
|
||||
'trello': ['TRELLO_API_KEY', 'TRELLO_TOKEN'],
|
||||
'jira': ['JIRA_TOKEN', 'JIRA_API_TOKEN'],
|
||||
'confluence': ['CONFLUENCE_TOKEN'],
|
||||
'bitbucket': ['BITBUCKET_TOKEN'],
|
||||
'dropbox': ['DROPBOX_ACCESS_TOKEN'],
|
||||
'box': ['BOX_ACCESS_TOKEN'],
|
||||
'spotify': ['SPOTIFY_CLIENT_ID', 'SPOTIFY_CLIENT_SECRET'],
|
||||
'youtube': ['YOUTUBE_API_KEY'],
|
||||
'twitter': ['TWITTER_API_KEY', 'TWITTER_BEARER_TOKEN'],
|
||||
'linkedin': ['LINKEDIN_ACCESS_TOKEN'],
|
||||
'facebook': ['FACEBOOK_ACCESS_TOKEN'],
|
||||
'instagram': ['INSTAGRAM_ACCESS_TOKEN'],
|
||||
'pinterest': ['PINTEREST_ACCESS_TOKEN'],
|
||||
'reddit': ['REDDIT_CLIENT_ID', 'REDDIT_CLIENT_SECRET'],
|
||||
'zendesk': ['ZENDESK_API_TOKEN'],
|
||||
'freshdesk': ['FRESHDESK_API_KEY'],
|
||||
'intercom': ['INTERCOM_ACCESS_TOKEN'],
|
||||
'mixpanel': ['MIXPANEL_TOKEN'],
|
||||
'segment': ['SEGMENT_WRITE_KEY'],
|
||||
'amplitude': ['AMPLITUDE_API_KEY'],
|
||||
'plaid': ['PLAID_CLIENT_ID', 'PLAID_SECRET'],
|
||||
'square': ['SQUARE_ACCESS_TOKEN'],
|
||||
'paypal': ['PAYPAL_CLIENT_ID', 'PAYPAL_CLIENT_SECRET'],
|
||||
'braintree': ['BRAINTREE_MERCHANT_ID'],
|
||||
'quickbooks': ['QUICKBOOKS_CLIENT_ID'],
|
||||
'xero': ['XERO_CLIENT_ID'],
|
||||
'salesforce': ['SALESFORCE_ACCESS_TOKEN'],
|
||||
'pipedrive': ['PIPEDRIVE_API_TOKEN'],
|
||||
'zoho': ['ZOHO_ACCESS_TOKEN'],
|
||||
'monday': ['MONDAY_API_KEY'],
|
||||
'clickup': ['CLICKUP_API_KEY'],
|
||||
'linear': ['LINEAR_API_KEY'],
|
||||
'vercel': ['VERCEL_TOKEN'],
|
||||
'netlify': ['NETLIFY_AUTH_TOKEN'],
|
||||
'heroku': ['HEROKU_API_KEY'],
|
||||
'digitalocean': ['DIGITALOCEAN_TOKEN', 'DO_TOKEN'],
|
||||
'linode': ['LINODE_TOKEN'],
|
||||
'vultr': ['VULTR_API_KEY'],
|
||||
'cloudflare': ['CLOUDFLARE_API_TOKEN', 'CF_API_TOKEN'],
|
||||
'datadog': ['DD_API_KEY', 'DATADOG_API_KEY'],
|
||||
'newrelic': ['NEW_RELIC_API_KEY'],
|
||||
'sentry': ['SENTRY_AUTH_TOKEN'],
|
||||
'pagerduty': ['PAGERDUTY_API_KEY'],
|
||||
'opsgenie': ['OPSGENIE_API_KEY'],
|
||||
'splunk': ['SPLUNK_TOKEN'],
|
||||
'elasticsearch': ['ELASTIC_API_KEY'],
|
||||
'algolia': ['ALGOLIA_API_KEY'],
|
||||
'meilisearch': ['MEILISEARCH_API_KEY'],
|
||||
'typesense': ['TYPESENSE_API_KEY'],
|
||||
'twitch': ['TWITCH_CLIENT_ID', 'TWITCH_CLIENT_SECRET'],
|
||||
'ebay': ['EBAY_APP_ID', 'EBAY_DEV_ID'],
|
||||
'etsy': ['ETSY_API_KEY'],
|
||||
'walmart': ['WALMART_CLIENT_ID'],
|
||||
'bestbuy': ['BESTBUY_API_KEY'],
|
||||
'yelp': ['YELP_API_KEY'],
|
||||
'foursquare': ['FOURSQUARE_API_KEY'],
|
||||
'tripadvisor': ['TRIPADVISOR_API_KEY'],
|
||||
'airbnb': ['AIRBNB_API_KEY'],
|
||||
'uber': ['UBER_ACCESS_TOKEN'],
|
||||
'lyft': ['LYFT_ACCESS_TOKEN'],
|
||||
'doordash': ['DOORDASH_API_KEY'],
|
||||
'postmates': ['POSTMATES_API_KEY'],
|
||||
'mapbox': ['MAPBOX_ACCESS_TOKEN'],
|
||||
'here': ['HERE_API_KEY'],
|
||||
'tomtom': ['TOMTOM_API_KEY'],
|
||||
'openweathermap': ['OPENWEATHERMAP_API_KEY'],
|
||||
'weatherapi': ['WEATHERAPI_KEY'],
|
||||
'newsapi': ['NEWSAPI_KEY'],
|
||||
'nytimes': ['NYTIMES_API_KEY'],
|
||||
'guardian': ['GUARDIAN_API_KEY'],
|
||||
'giphy': ['GIPHY_API_KEY'],
|
||||
'unsplash': ['UNSPLASH_ACCESS_KEY'],
|
||||
'pexels': ['PEXELS_API_KEY'],
|
||||
'cloudinary': ['CLOUDINARY_API_KEY'],
|
||||
'imgix': ['IMGIX_API_KEY'],
|
||||
'uploadcare': ['UPLOADCARE_PUBLIC_KEY'],
|
||||
'filestack': ['FILESTACK_API_KEY'],
|
||||
'agora': ['AGORA_APP_ID'],
|
||||
'vonage': ['VONAGE_API_KEY', 'NEXMO_API_KEY'],
|
||||
'messagebird': ['MESSAGEBIRD_API_KEY'],
|
||||
'bandwidth': ['BANDWIDTH_API_TOKEN'],
|
||||
'telnyx': ['TELNYX_API_KEY'],
|
||||
'apilayer': ['APILAYER_API_KEY'],
|
||||
'currencylayer': ['CURRENCYLAYER_API_KEY'],
|
||||
'exchangerate': ['EXCHANGERATE_API_KEY'],
|
||||
'coinbase': ['COINBASE_API_KEY'],
|
||||
'binance': ['BINANCE_API_KEY'],
|
||||
'kraken': ['KRAKEN_API_KEY'],
|
||||
'alchemy': ['ALCHEMY_API_KEY'],
|
||||
'infura': ['INFURA_PROJECT_ID'],
|
||||
'moralis': ['MORALIS_API_KEY'],
|
||||
'thegraph': ['THEGRAPH_API_KEY'],
|
||||
'openai': ['OPENAI_API_KEY'],
|
||||
'anthropic': ['ANTHROPIC_API_KEY'],
|
||||
'cohere': ['COHERE_API_KEY'],
|
||||
'huggingface': ['HF_TOKEN', 'HUGGINGFACE_TOKEN'],
|
||||
'replicate': ['REPLICATE_API_TOKEN'],
|
||||
'stability': ['STABILITY_API_KEY'],
|
||||
'deepl': ['DEEPL_API_KEY'],
|
||||
'ably': ['ABLY_API_KEY'],
|
||||
'pusher': ['PUSHER_APP_KEY'],
|
||||
'pubnub': ['PUBNUB_SUBSCRIBE_KEY'],
|
||||
'firebase': ['FIREBASE_API_KEY'],
|
||||
'supabase': ['SUPABASE_API_KEY'],
|
||||
'mongodb': ['MONGODB_API_KEY'],
|
||||
'redis': ['REDIS_API_KEY'],
|
||||
'cockroachdb': ['COCKROACHDB_API_KEY'],
|
||||
'planetscale': ['PLANETSCALE_TOKEN'],
|
||||
'neon': ['NEON_API_KEY'],
|
||||
'upstash': ['UPSTASH_API_KEY'],
|
||||
'fauna': ['FAUNA_SECRET'],
|
||||
'adyen': ['ADYEN_API_KEY'],
|
||||
'sportsdata': ['SPORTSDATA_API_KEY'],
|
||||
'amadeus': ['AMADEUS_API_KEY', 'AMADEUS_API_SECRET'],
|
||||
'nexmo': ['NEXMO_API_KEY', 'NEXMO_API_SECRET'],
|
||||
'mastercard': ['MASTERCARD_API_KEY'],
|
||||
'hubapi': ['HUBSPOT_API_KEY'],
|
||||
'apideck': ['APIDECK_API_KEY'],
|
||||
'codat': ['CODAT_API_KEY'],
|
||||
'deutschebahn': ['DEUTSCHEBAHN_API_KEY'],
|
||||
'rapidapi': ['RAPIDAPI_KEY'],
|
||||
'whapi': ['WHAPI_TOKEN'],
|
||||
'vtex': ['VTEX_APP_KEY', 'VTEX_APP_TOKEN'],
|
||||
'interzoid': ['INTERZOID_API_KEY'],
|
||||
"stripe": ["STRIPE_API_KEY", "STRIPE_SECRET_KEY"],
|
||||
"twilio": ["TWILIO_ACCOUNT_SID", "TWILIO_AUTH_TOKEN"],
|
||||
"github": ["GITHUB_TOKEN", "GH_TOKEN"],
|
||||
"gitlab": ["GITLAB_TOKEN"],
|
||||
"slack": ["SLACK_TOKEN", "SLACK_BOT_TOKEN"],
|
||||
"discord": ["DISCORD_TOKEN", "DISCORD_BOT_TOKEN"],
|
||||
"shopify": ["SHOPIFY_API_KEY", "SHOPIFY_ACCESS_TOKEN"],
|
||||
"hubspot": ["HUBSPOT_API_KEY", "HUBSPOT_ACCESS_TOKEN"],
|
||||
"mailchimp": ["MAILCHIMP_API_KEY"],
|
||||
"sendgrid": ["SENDGRID_API_KEY"],
|
||||
"mailgun": ["MAILGUN_API_KEY"],
|
||||
"contentful": ["CONTENTFUL_ACCESS_TOKEN"],
|
||||
"airtable": ["AIRTABLE_API_KEY"],
|
||||
"notion": ["NOTION_API_KEY", "NOTION_TOKEN"],
|
||||
"figma": ["FIGMA_TOKEN", "FIGMA_ACCESS_TOKEN"],
|
||||
"asana": ["ASANA_TOKEN", "ASANA_ACCESS_TOKEN"],
|
||||
"trello": ["TRELLO_API_KEY", "TRELLO_TOKEN"],
|
||||
"jira": ["JIRA_TOKEN", "JIRA_API_TOKEN"],
|
||||
"confluence": ["CONFLUENCE_TOKEN"],
|
||||
"bitbucket": ["BITBUCKET_TOKEN"],
|
||||
"dropbox": ["DROPBOX_ACCESS_TOKEN"],
|
||||
"box": ["BOX_ACCESS_TOKEN"],
|
||||
"spotify": ["SPOTIFY_CLIENT_ID", "SPOTIFY_CLIENT_SECRET"],
|
||||
"youtube": ["YOUTUBE_API_KEY"],
|
||||
"twitter": ["TWITTER_API_KEY", "TWITTER_BEARER_TOKEN"],
|
||||
"linkedin": ["LINKEDIN_ACCESS_TOKEN"],
|
||||
"facebook": ["FACEBOOK_ACCESS_TOKEN"],
|
||||
"instagram": ["INSTAGRAM_ACCESS_TOKEN"],
|
||||
"pinterest": ["PINTEREST_ACCESS_TOKEN"],
|
||||
"reddit": ["REDDIT_CLIENT_ID", "REDDIT_CLIENT_SECRET"],
|
||||
"zendesk": ["ZENDESK_API_TOKEN"],
|
||||
"freshdesk": ["FRESHDESK_API_KEY"],
|
||||
"intercom": ["INTERCOM_ACCESS_TOKEN"],
|
||||
"mixpanel": ["MIXPANEL_TOKEN"],
|
||||
"segment": ["SEGMENT_WRITE_KEY"],
|
||||
"amplitude": ["AMPLITUDE_API_KEY"],
|
||||
"plaid": ["PLAID_CLIENT_ID", "PLAID_SECRET"],
|
||||
"square": ["SQUARE_ACCESS_TOKEN"],
|
||||
"paypal": ["PAYPAL_CLIENT_ID", "PAYPAL_CLIENT_SECRET"],
|
||||
"braintree": ["BRAINTREE_MERCHANT_ID"],
|
||||
"quickbooks": ["QUICKBOOKS_CLIENT_ID"],
|
||||
"xero": ["XERO_CLIENT_ID"],
|
||||
"salesforce": ["SALESFORCE_ACCESS_TOKEN"],
|
||||
"pipedrive": ["PIPEDRIVE_API_TOKEN"],
|
||||
"zoho": ["ZOHO_ACCESS_TOKEN"],
|
||||
"monday": ["MONDAY_API_KEY"],
|
||||
"clickup": ["CLICKUP_API_KEY"],
|
||||
"linear": ["LINEAR_API_KEY"],
|
||||
"vercel": ["VERCEL_TOKEN"],
|
||||
"netlify": ["NETLIFY_AUTH_TOKEN"],
|
||||
"heroku": ["HEROKU_API_KEY"],
|
||||
"digitalocean": ["DIGITALOCEAN_TOKEN", "DO_TOKEN"],
|
||||
"linode": ["LINODE_TOKEN"],
|
||||
"vultr": ["VULTR_API_KEY"],
|
||||
"cloudflare": ["CLOUDFLARE_API_TOKEN", "CF_API_TOKEN"],
|
||||
"datadog": ["DD_API_KEY", "DATADOG_API_KEY"],
|
||||
"newrelic": ["NEW_RELIC_API_KEY"],
|
||||
"sentry": ["SENTRY_AUTH_TOKEN"],
|
||||
"pagerduty": ["PAGERDUTY_API_KEY"],
|
||||
"opsgenie": ["OPSGENIE_API_KEY"],
|
||||
"splunk": ["SPLUNK_TOKEN"],
|
||||
"elasticsearch": ["ELASTIC_API_KEY"],
|
||||
"algolia": ["ALGOLIA_API_KEY"],
|
||||
"meilisearch": ["MEILISEARCH_API_KEY"],
|
||||
"typesense": ["TYPESENSE_API_KEY"],
|
||||
"twitch": ["TWITCH_CLIENT_ID", "TWITCH_CLIENT_SECRET"],
|
||||
"ebay": ["EBAY_APP_ID", "EBAY_DEV_ID"],
|
||||
"etsy": ["ETSY_API_KEY"],
|
||||
"walmart": ["WALMART_CLIENT_ID"],
|
||||
"bestbuy": ["BESTBUY_API_KEY"],
|
||||
"yelp": ["YELP_API_KEY"],
|
||||
"foursquare": ["FOURSQUARE_API_KEY"],
|
||||
"tripadvisor": ["TRIPADVISOR_API_KEY"],
|
||||
"airbnb": ["AIRBNB_API_KEY"],
|
||||
"uber": ["UBER_ACCESS_TOKEN"],
|
||||
"lyft": ["LYFT_ACCESS_TOKEN"],
|
||||
"doordash": ["DOORDASH_API_KEY"],
|
||||
"postmates": ["POSTMATES_API_KEY"],
|
||||
"mapbox": ["MAPBOX_ACCESS_TOKEN"],
|
||||
"here": ["HERE_API_KEY"],
|
||||
"tomtom": ["TOMTOM_API_KEY"],
|
||||
"openweathermap": ["OPENWEATHERMAP_API_KEY"],
|
||||
"weatherapi": ["WEATHERAPI_KEY"],
|
||||
"newsapi": ["NEWSAPI_KEY"],
|
||||
"nytimes": ["NYTIMES_API_KEY"],
|
||||
"guardian": ["GUARDIAN_API_KEY"],
|
||||
"giphy": ["GIPHY_API_KEY"],
|
||||
"unsplash": ["UNSPLASH_ACCESS_KEY"],
|
||||
"pexels": ["PEXELS_API_KEY"],
|
||||
"cloudinary": ["CLOUDINARY_API_KEY"],
|
||||
"imgix": ["IMGIX_API_KEY"],
|
||||
"uploadcare": ["UPLOADCARE_PUBLIC_KEY"],
|
||||
"filestack": ["FILESTACK_API_KEY"],
|
||||
"agora": ["AGORA_APP_ID"],
|
||||
"vonage": ["VONAGE_API_KEY", "NEXMO_API_KEY"],
|
||||
"messagebird": ["MESSAGEBIRD_API_KEY"],
|
||||
"bandwidth": ["BANDWIDTH_API_TOKEN"],
|
||||
"telnyx": ["TELNYX_API_KEY"],
|
||||
"apilayer": ["APILAYER_API_KEY"],
|
||||
"currencylayer": ["CURRENCYLAYER_API_KEY"],
|
||||
"exchangerate": ["EXCHANGERATE_API_KEY"],
|
||||
"coinbase": ["COINBASE_API_KEY"],
|
||||
"binance": ["BINANCE_API_KEY"],
|
||||
"kraken": ["KRAKEN_API_KEY"],
|
||||
"alchemy": ["ALCHEMY_API_KEY"],
|
||||
"infura": ["INFURA_PROJECT_ID"],
|
||||
"moralis": ["MORALIS_API_KEY"],
|
||||
"thegraph": ["THEGRAPH_API_KEY"],
|
||||
"openai": ["OPENAI_API_KEY"],
|
||||
"anthropic": ["ANTHROPIC_API_KEY"],
|
||||
"cohere": ["COHERE_API_KEY"],
|
||||
"huggingface": ["HF_TOKEN", "HUGGINGFACE_TOKEN"],
|
||||
"replicate": ["REPLICATE_API_TOKEN"],
|
||||
"stability": ["STABILITY_API_KEY"],
|
||||
"deepl": ["DEEPL_API_KEY"],
|
||||
"ably": ["ABLY_API_KEY"],
|
||||
"pusher": ["PUSHER_APP_KEY"],
|
||||
"pubnub": ["PUBNUB_SUBSCRIBE_KEY"],
|
||||
"firebase": ["FIREBASE_API_KEY"],
|
||||
"supabase": ["SUPABASE_API_KEY"],
|
||||
"mongodb": ["MONGODB_API_KEY"],
|
||||
"redis": ["REDIS_API_KEY"],
|
||||
"cockroachdb": ["COCKROACHDB_API_KEY"],
|
||||
"planetscale": ["PLANETSCALE_TOKEN"],
|
||||
"neon": ["NEON_API_KEY"],
|
||||
"upstash": ["UPSTASH_API_KEY"],
|
||||
"fauna": ["FAUNA_SECRET"],
|
||||
"adyen": ["ADYEN_API_KEY"],
|
||||
"sportsdata": ["SPORTSDATA_API_KEY"],
|
||||
"amadeus": ["AMADEUS_API_KEY", "AMADEUS_API_SECRET"],
|
||||
"nexmo": ["NEXMO_API_KEY", "NEXMO_API_SECRET"],
|
||||
"mastercard": ["MASTERCARD_API_KEY"],
|
||||
"hubapi": ["HUBSPOT_API_KEY"],
|
||||
"apideck": ["APIDECK_API_KEY"],
|
||||
"codat": ["CODAT_API_KEY"],
|
||||
"deutschebahn": ["DEUTSCHEBAHN_API_KEY"],
|
||||
"rapidapi": ["RAPIDAPI_KEY"],
|
||||
"whapi": ["WHAPI_TOKEN"],
|
||||
"vtex": ["VTEX_APP_KEY", "VTEX_APP_TOKEN"],
|
||||
"interzoid": ["INTERZOID_API_KEY"],
|
||||
}
|
||||
|
||||
# Skip cloud-specific APIs that need special auth handling
|
||||
SKIP_PREFIXES = [
|
||||
'amazonaws', 'azure', 'googleapis', 'google.',
|
||||
'apisetu', 'parliament', 'gov.', 'opto22',
|
||||
'windows', 'o365', 'microsofthealth',
|
||||
"amazonaws",
|
||||
"azure",
|
||||
"googleapis",
|
||||
"google.",
|
||||
"apisetu",
|
||||
"parliament",
|
||||
"gov.",
|
||||
"opto22",
|
||||
"windows",
|
||||
"o365",
|
||||
"microsofthealth",
|
||||
]
|
||||
|
||||
# Popular APIs with oapis.org LLM-friendly descriptions
|
||||
# These get priority loading and better descriptions
|
||||
OAPIS_POPULAR = [
|
||||
'github', 'stripe', 'twilio', 'slack', 'notion', 'discord',
|
||||
'shopify', 'spotify', 'twitter', 'openai', 'anthropic',
|
||||
"github",
|
||||
"stripe",
|
||||
"twilio",
|
||||
"slack",
|
||||
"notion",
|
||||
"discord",
|
||||
"shopify",
|
||||
"spotify",
|
||||
"twitter",
|
||||
"openai",
|
||||
"anthropic",
|
||||
]
|
||||
|
||||
|
||||
@@ -175,21 +190,21 @@ def get_env_vars(name: str) -> list[str]:
|
||||
if key in name_lower:
|
||||
return vars
|
||||
# Generate default pattern
|
||||
clean = re.sub(r'[^a-z0-9]', '_', name_lower).upper()
|
||||
return [f'{clean}_API_KEY', f'{clean}_TOKEN']
|
||||
clean = re.sub(r"[^a-z0-9]", "_", name_lower).upper()
|
||||
return [f"{clean}_API_KEY", f"{clean}_TOKEN"]
|
||||
|
||||
|
||||
def clean_name(name: str) -> str:
|
||||
"""Clean API name for use as provider ID."""
|
||||
# Remove version suffixes
|
||||
name = re.sub(r':\d+.*$', '', name)
|
||||
name = re.sub(r":\d+.*$", "", name)
|
||||
# Replace dots and colons with hyphens
|
||||
name = name.replace('.', '-').replace(':', '-')
|
||||
name = name.replace(".", "-").replace(":", "-")
|
||||
# Remove common suffixes
|
||||
name = re.sub(r'-com$|-io$|-net$|-org$|-local$', '', name)
|
||||
name = re.sub(r"-com$|-io$|-net$|-org$|-local$", "", name)
|
||||
# Clean up
|
||||
name = re.sub(r'[^a-z0-9-]', '', name.lower())
|
||||
name = re.sub(r'-+', '-', name).strip('-')
|
||||
name = re.sub(r"[^a-z0-9-]", "", name.lower())
|
||||
name = re.sub(r"-+", "-", name).strip("-")
|
||||
return name
|
||||
|
||||
|
||||
@@ -200,7 +215,7 @@ def fetch_oapis_slop(name: str) -> dict | None:
|
||||
"""
|
||||
try:
|
||||
url = f"https://oapis.org/slop/{name}"
|
||||
req = urllib.request.Request(url, headers={'User-Agent': 'hanzo-tools-api/0.2'})
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "hanzo-tools-api/0.2"})
|
||||
with urllib.request.urlopen(req, timeout=10) as response:
|
||||
content = response.read().decode()
|
||||
|
||||
@@ -208,21 +223,21 @@ def fetch_oapis_slop(name: str) -> dict | None:
|
||||
result = {}
|
||||
|
||||
# Extract base URL (usually on a line with https://)
|
||||
base_match = re.search(r'Base URL:\s*(https?://[^\s]+)', content, re.IGNORECASE)
|
||||
base_match = re.search(r"Base URL:\s*(https?://[^\s]+)", content, re.IGNORECASE)
|
||||
if not base_match:
|
||||
base_match = re.search(r'\*\*Base URL\*\*:\s*(https?://[^\s]+)', content, re.IGNORECASE)
|
||||
base_match = re.search(r"\*\*Base URL\*\*:\s*(https?://[^\s]+)", content, re.IGNORECASE)
|
||||
if base_match:
|
||||
result['base_url'] = base_match.group(1).rstrip('/')
|
||||
result["base_url"] = base_match.group(1).rstrip("/")
|
||||
|
||||
# Extract endpoint count
|
||||
count_match = re.search(r'(\d+)\s+endpoints?', content, re.IGNORECASE)
|
||||
count_match = re.search(r"(\d+)\s+endpoints?", content, re.IGNORECASE)
|
||||
if count_match:
|
||||
result['endpoints_count'] = int(count_match.group(1))
|
||||
result["endpoints_count"] = int(count_match.group(1))
|
||||
|
||||
# Extract description (first paragraph after title)
|
||||
desc_match = re.search(r'^#[^\n]+\n+([^#\n][^\n]+)', content, re.MULTILINE)
|
||||
desc_match = re.search(r"^#[^\n]+\n+([^#\n][^\n]+)", content, re.MULTILINE)
|
||||
if desc_match:
|
||||
result['description'] = desc_match.group(1).strip()[:200]
|
||||
result["description"] = desc_match.group(1).strip()[:200]
|
||||
|
||||
return result if result else None
|
||||
except Exception:
|
||||
@@ -274,17 +289,17 @@ def main():
|
||||
if skip:
|
||||
continue
|
||||
|
||||
preferred = info.get('preferred', '')
|
||||
if not preferred or preferred not in info.get('versions', {}):
|
||||
preferred = info.get("preferred", "")
|
||||
if not preferred or preferred not in info.get("versions", {}):
|
||||
continue
|
||||
|
||||
version_info = info['versions'][preferred]
|
||||
spec_url = version_info.get('swaggerUrl', '')
|
||||
version_info = info["versions"][preferred]
|
||||
spec_url = version_info.get("swaggerUrl", "")
|
||||
if not spec_url:
|
||||
continue
|
||||
|
||||
api_info = version_info.get('info', {})
|
||||
title = api_info.get('title', name)
|
||||
api_info = version_info.get("info", {})
|
||||
title = api_info.get("title", name)
|
||||
|
||||
clean = clean_name(name)
|
||||
if clean in seen_names or len(clean) < 2:
|
||||
@@ -292,11 +307,11 @@ def main():
|
||||
seen_names.add(clean)
|
||||
|
||||
# Get base URL from x-origin
|
||||
base_url = ''
|
||||
if 'x-origin' in api_info:
|
||||
x_origin = api_info['x-origin']
|
||||
base_url = ""
|
||||
if "x-origin" in api_info:
|
||||
x_origin = api_info["x-origin"]
|
||||
if isinstance(x_origin, list) and x_origin:
|
||||
base_url = x_origin[0].get('url', '')
|
||||
base_url = x_origin[0].get("url", "")
|
||||
|
||||
# Check for oapis.org enhanced data
|
||||
oapis_key = None
|
||||
@@ -305,81 +320,83 @@ def main():
|
||||
oapis_key = popular
|
||||
break
|
||||
|
||||
description = api_info.get('description', '')[:200] if api_info.get('description') else ''
|
||||
description = api_info.get("description", "")[:200] if api_info.get("description") else ""
|
||||
endpoints_count = None
|
||||
|
||||
if oapis_key and oapis_key in oapis_data:
|
||||
oapis_info = oapis_data[oapis_key]
|
||||
if oapis_info.get('base_url'):
|
||||
base_url = oapis_info['base_url']
|
||||
if oapis_info.get('description'):
|
||||
description = oapis_info['description']
|
||||
if oapis_info.get('endpoints_count'):
|
||||
endpoints_count = oapis_info['endpoints_count']
|
||||
if oapis_info.get("base_url"):
|
||||
base_url = oapis_info["base_url"]
|
||||
if oapis_info.get("description"):
|
||||
description = oapis_info["description"]
|
||||
if oapis_info.get("endpoints_count"):
|
||||
endpoints_count = oapis_info["endpoints_count"]
|
||||
|
||||
# Clean strings - remove newlines, escape quotes, limit length
|
||||
def clean_str(s: str, max_len: int = 150) -> str:
|
||||
if not s:
|
||||
return ""
|
||||
# Replace various whitespace with single space
|
||||
s = ' '.join(s.split())
|
||||
s = " ".join(s.split())
|
||||
# Escape backslashes first, then quotes
|
||||
s = s.replace('\\', '\\\\').replace('"', '\\"')
|
||||
s = s.replace("\\", "\\\\").replace('"', '\\"')
|
||||
return s[:max_len]
|
||||
|
||||
apis.append({
|
||||
'name': clean,
|
||||
'title': clean_str(title, 60),
|
||||
'description': clean_str(description, 150),
|
||||
'spec_url': spec_url,
|
||||
'base_url': base_url,
|
||||
'env_vars': get_env_vars(clean),
|
||||
'endpoints_count': endpoints_count,
|
||||
})
|
||||
apis.append(
|
||||
{
|
||||
"name": clean,
|
||||
"title": clean_str(title, 60),
|
||||
"description": clean_str(description, 150),
|
||||
"spec_url": spec_url,
|
||||
"base_url": base_url,
|
||||
"env_vars": get_env_vars(clean),
|
||||
"endpoints_count": endpoints_count,
|
||||
}
|
||||
)
|
||||
|
||||
# Sort by name
|
||||
apis.sort(key=lambda x: x['name'])
|
||||
apis.sort(key=lambda x: x["name"])
|
||||
|
||||
# Generate Python code
|
||||
print('"""Auto-generated provider configurations from APIs.guru + oapis.org.')
|
||||
print()
|
||||
print(f'Generated {len(apis)} provider configurations.')
|
||||
print('Includes LLM-optimized descriptions from oapis.org for popular APIs.')
|
||||
print(f"Generated {len(apis)} provider configurations.")
|
||||
print("Includes LLM-optimized descriptions from oapis.org for popular APIs.")
|
||||
print()
|
||||
print('Regenerate with: python scripts/generate_providers.py > hanzo_tools/api/apis_guru_providers.py')
|
||||
print("Regenerate with: python scripts/generate_providers.py > hanzo_tools/api/apis_guru_providers.py")
|
||||
print('"""')
|
||||
print()
|
||||
print('from typing import Any')
|
||||
print("from typing import Any")
|
||||
print()
|
||||
print()
|
||||
print('# =============================================================================')
|
||||
print('# APIs.guru Provider Configurations (with oapis.org enhancements)')
|
||||
print('# =============================================================================')
|
||||
print("# =============================================================================")
|
||||
print("# APIs.guru Provider Configurations (with oapis.org enhancements)")
|
||||
print("# =============================================================================")
|
||||
print()
|
||||
print('APIS_GURU_PROVIDERS: dict[str, dict[str, Any]] = {')
|
||||
print("APIS_GURU_PROVIDERS: dict[str, dict[str, Any]] = {")
|
||||
|
||||
for api in apis:
|
||||
env_str = ', '.join(f'"{v}"' for v in api['env_vars'][:2])
|
||||
env_str = ", ".join(f'"{v}"' for v in api["env_vars"][:2])
|
||||
print(f' "{api["name"]}": {{')
|
||||
print(f' "display_name": "{api["title"]}",')
|
||||
if api['description']:
|
||||
if api["description"]:
|
||||
print(f' "description": "{api["description"][:150]}",')
|
||||
print(f' "spec_url": "{api["spec_url"]}",')
|
||||
if api['base_url']:
|
||||
safe_url = api['base_url'].replace('"', '\\"')
|
||||
if api["base_url"]:
|
||||
safe_url = api["base_url"].replace('"', '\\"')
|
||||
print(f' "base_url": "{safe_url}",')
|
||||
print(f' "env_vars": [{env_str}],')
|
||||
if api['endpoints_count']:
|
||||
if api["endpoints_count"]:
|
||||
print(f' "endpoints_count": {api["endpoints_count"]},')
|
||||
print(' },')
|
||||
print(" },")
|
||||
|
||||
print('}')
|
||||
print("}")
|
||||
print()
|
||||
print()
|
||||
print(f'# Total: {len(apis)} providers')
|
||||
print(f"# Total: {len(apis)} providers")
|
||||
|
||||
print(f"\nGenerated {len(apis)} providers", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -8,8 +8,8 @@ Usage:
|
||||
python scripts/preload_specs.py github stripe # Download specific specs
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import argparse
|
||||
import asyncio
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
@@ -17,28 +17,65 @@ from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from hanzo_tools.api.client import APIClient
|
||||
from hanzo_tools.api.providers import list_providers_with_specs, get_provider_config
|
||||
from hanzo_tools.api.providers import get_provider_config, list_providers_with_specs
|
||||
|
||||
# Top 50 most popular APIs to pre-cache
|
||||
POPULAR_APIS = [
|
||||
# AI/ML
|
||||
"openai", "anthropic", "together", "groq", "mistral", "cohere", "replicate",
|
||||
"openai",
|
||||
"anthropic",
|
||||
"together",
|
||||
"groq",
|
||||
"mistral",
|
||||
"cohere",
|
||||
"replicate",
|
||||
# Developer
|
||||
"github", "gitlab", "bitbucket", "vercel", "netlify", "fly", "railway", "render",
|
||||
"github",
|
||||
"gitlab",
|
||||
"bitbucket",
|
||||
"vercel",
|
||||
"netlify",
|
||||
"fly",
|
||||
"railway",
|
||||
"render",
|
||||
# Cloud
|
||||
"cloudflare", "digitalocean", "hetzner", "linode", "vultr",
|
||||
"cloudflare",
|
||||
"digitalocean",
|
||||
"hetzner",
|
||||
"linode",
|
||||
"vultr",
|
||||
# Payment
|
||||
"stripe", "paypal", "adyen-com-accountservice", "square",
|
||||
"stripe",
|
||||
"paypal",
|
||||
"adyen-com-accountservice",
|
||||
"square",
|
||||
# Communication
|
||||
"twilio", "sendgrid", "resend", "slack", "discord",
|
||||
"twilio",
|
||||
"sendgrid",
|
||||
"resend",
|
||||
"slack",
|
||||
"discord",
|
||||
# Databases
|
||||
"supabase", "neon", "planetscale", "mongodb", "redis",
|
||||
"supabase",
|
||||
"neon",
|
||||
"planetscale",
|
||||
"mongodb",
|
||||
"redis",
|
||||
# Search
|
||||
"algolia", "elasticsearch", "meilisearch", "typesense",
|
||||
"algolia",
|
||||
"elasticsearch",
|
||||
"meilisearch",
|
||||
"typesense",
|
||||
# Monitoring
|
||||
"datadog", "sentry", "newrelic",
|
||||
"datadog",
|
||||
"sentry",
|
||||
"newrelic",
|
||||
# Other popular
|
||||
"notion", "airtable", "hubspot", "shopify", "jira",
|
||||
"notion",
|
||||
"airtable",
|
||||
"hubspot",
|
||||
"shopify",
|
||||
"jira",
|
||||
]
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user