mirror of
https://github.com/hanzoai/site.git
synced 2026-11-20 21:14:28 +00:00
55 lines
2.5 KiB
Python
55 lines
2.5 KiB
Python
"""Outline every ported page: its headings, paragraphs, lists, links and media, section by section.
|
|
The outline is what a page is rebuilt from, so every word and link on the new page is Hanzo's.
|
|
Run: python3 scripts/outline.py -> content/outline/<path>.json
|
|
"""
|
|
import json, os, re
|
|
from bs4 import BeautifulSoup
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
SRC, OUT = os.path.join(ROOT, 'content', 'pages'), os.path.join(ROOT, 'content', 'outline')
|
|
|
|
def walk(node, out):
|
|
for el in node.find_all(['h1', 'h2', 'h3', 'h4', 'p', 'li', 'a', 'img', 'video', 'pre', 'code', 'table', 'button'], recursive=True):
|
|
if el.find_parent(['a', 'li', 'pre', 'table', 'button']) and el.name not in ('img',):
|
|
continue # counted with its parent
|
|
t = el.get_text(' ', strip=True)
|
|
if el.name in ('h1', 'h2', 'h3', 'h4'):
|
|
out.append({el.name: t})
|
|
elif el.name == 'p' and t:
|
|
out.append({'p': t})
|
|
elif el.name == 'li' and t:
|
|
a = el.find('a')
|
|
out.append({'li': t, **({'href': a.get('href')} if a else {})})
|
|
elif el.name == 'a' and t:
|
|
out.append({'a': t, 'href': el.get('href')})
|
|
elif el.name == 'button' and t:
|
|
out.append({'button': t})
|
|
elif el.name in ('img', 'video') and (el.get('src') or el.get('poster')):
|
|
out.append({el.name: el.get('src') or el.get('poster'), 'alt': el.get('alt', '')})
|
|
elif el.name in ('pre', 'code') and t:
|
|
out.append({'code': t[:600]})
|
|
elif el.name == 'table':
|
|
out.append({'table': [[c.get_text(' ', strip=True) for c in r.find_all(['th', 'td'])] for r in el.find_all('tr')][:40]})
|
|
|
|
if __name__ == '__main__':
|
|
n = 0
|
|
for dp, _, fs in os.walk(SRC):
|
|
for f in fs:
|
|
if not f.endswith('.html'):
|
|
continue
|
|
src = os.path.join(dp, f)
|
|
rel = os.path.relpath(src, SRC)[:-5]
|
|
s = BeautifulSoup(open(src).read(), 'html.parser')
|
|
secs = [x for x in s.find_all('section') if not x.find_parent('section')] or [s]
|
|
data = []
|
|
for x in secs:
|
|
items = []
|
|
walk(x, items)
|
|
if items:
|
|
data.append(items)
|
|
dst = os.path.join(OUT, rel + '.json')
|
|
os.makedirs(os.path.dirname(dst), exist_ok=True)
|
|
json.dump(data, open(dst, 'w'), ensure_ascii=False, indent=0)
|
|
n += 1
|
|
print('outlined', n)
|