mirror of
https://github.com/hanzoai/site.git
synced 2026-11-20 21:11:05 +00:00
32 lines
1.5 KiB
Python
32 lines
1.5 KiB
Python
"""Keep Hanzo's structured data (schema.org JSON-LD) for every page: search engines read it for FAQs, products, breadcrumbs.
|
|
Run: python3 scripts/jsonld.py -> content/jsonld/<path>.json (list of JSON-LD objects, home at content/jsonld/index.json)
|
|
"""
|
|
import json, os, re, urllib.request
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
OUT = os.path.join(ROOT, 'content', 'jsonld')
|
|
UA = {'User-Agent': 'Mozilla/5.0 (Macintosh) AppleWebKit/537.36 Chrome/130 Safari/537.36'}
|
|
|
|
def grab(path):
|
|
try:
|
|
h = urllib.request.urlopen(urllib.request.Request('https://hanzo.ai' + path, headers=UA), timeout=40).read().decode('utf-8', 'replace')
|
|
except Exception as e:
|
|
return path, None
|
|
items = []
|
|
for m in re.finditer(r'<script[^>]*type="application/ld\+json"[^>]*>(.*?)</script>', h, re.S):
|
|
try:
|
|
items.append(json.loads(m.group(1)))
|
|
except ValueError:
|
|
pass
|
|
f = os.path.join(OUT, (path.strip('/') or 'index') + '.json')
|
|
os.makedirs(os.path.dirname(f), exist_ok=True)
|
|
json.dump(items, open(f, 'w'), ensure_ascii=False)
|
|
return path, len(items)
|
|
|
|
if __name__ == '__main__':
|
|
paths = ['/'] + list(json.load(open(os.path.join(ROOT, 'content', 'pages.json'))))
|
|
with ThreadPoolExecutor(8) as ex:
|
|
res = list(ex.map(grab, paths))
|
|
print('pages', len(res), 'with data', sum(1 for _, n in res if n), 'failed', sum(1 for _, n in res if n is None))
|