mirror of
https://github.com/hanzoai/site.git
synced 2026-11-21 01:07:08 +00:00
live.mjs and openSource/read.mjs fetched hanzo.ai's deployed JavaScript and lifted the JSON literal out of it: the home band's tiles, calendar and cards, and /open-source's boards. That bundle goes away at cutover, and it was two days old when read. lib/oss.mjs ports hanzo-inc/app's scripts/sync-oss.ts. The estate (the year, the days, the months, the forge) is GET /v1/git/activity, read every build; GitHub's half (stars, forks, languages, contributors, the cards, each org's description) comes from GitHub when GITHUB_TOKEN is set, else from content/live/_github.json, and a push reuses the contributors and descriptions it holds. hanzo.yml hands the runner's GIT_TOKEN over as GITHUB_TOKEN. The numbers are the app's: Hanzo 5,149 stars against 5,130 two days earlier, the same 16, 6 and 6 cards, 38, 15 and 4 organizations. read.mjs is now shape.mjs, which only shapes; live.mjs writes both _oss.json and snapshot.js from the one read. The cards for hanzoai/ai and hanzoai/cloud, which the forge does not list, had no address and linked "undefined"; they open GitHub, as the app's cards do. Claude-Session: https://claude.ai/code/session_015XYKLix9PSyn6Jw2HWKprk
214 lines
15 KiB
JavaScript
214 lines
15 KiB
JavaScript
// The repositories Hanzo, Lux and Zoo publish and the work done in them: hanzo-inc/app's scripts/sync-oss.ts, for the two
|
|
// pages that show them (the home's open-source band and /open-source). Three steps, kept apart because they cost
|
|
// differently, and nothing here reads a page of hanzo.ai:
|
|
// estate() GET api.hanzo.ai/v1/git/activity, no key: the year's commits and merges, the days, the latest commits, the
|
|
// months of every org and repository, and the forge's own figures. Every build reads it.
|
|
// github() GET api.github.com/orgs/<org>/repos and what only GitHub knows: stars, forks, languages, contributors, an
|
|
// org's own description. About 150 calls, so it takes a token (GITHUB_TOKEN); a push reuses the contributors,
|
|
// sites and org descriptions of the last read it holds.
|
|
// combine() the two as sync-oss.ts writes them (oss-stats.json and oss-activity.json), with no network.
|
|
|
|
const ACTIVITY = 'https://api.hanzo.ai/v1/git/activity';
|
|
const GITHUB = 'https://api.github.com';
|
|
const FORGE = 'https://git.hanzo.ai';
|
|
const DAY = 864e5;
|
|
|
|
// Each group's name, the org its GitHub link opens and the repositories it leads with: the site's own editorial choices.
|
|
// The orgs of a group are the estate's: the ones GitHub holds.
|
|
export const GROUPS = {
|
|
hanzo: { name: 'Hanzo', lead: 'hanzoai', featured: ['hanzoai/ai', 'hanzoai/engine', 'hanzoai/cloud', 'hanzoai/dev', 'hanzoai/cli', 'hanzoai/mcp', 'hanzoai/bot', 'hanzoai/ui', 'hanzoai/iam', 'hanzoai/ml', 'hanzoai/chat', 'hanzoai/gui', 'hanzoai/base', 'hanzoai/kms', 'hanzoai/flow', 'hanzoai/insights'] },
|
|
lux: { name: 'Lux', lead: 'luxfi', featured: ['luxfi/node', 'luxfi/consensus', 'luxfi/evm', 'luxfi/threshold', 'luxfi/cli', 'luxfhe/torus'] },
|
|
zoo: { name: 'Zoo', lead: 'zooai', featured: ['zooai/ai', 'zooai/node', 'zooai/ngo', 'zooai/proofs', 'zooai/cli', 'zooai/evm'] },
|
|
};
|
|
// implementation repositories: counted in the totals, never shown as cards
|
|
const UNLISTED = new Set(['hanzoai/laya', 'hanzoai/sensei.group', 'hanzo-apps/sensei.group', 'hanzo-community/hanzo_sensei-group']);
|
|
|
|
// A description as these pages print it (hanzo-inc/app lib/upstream.ts and scripts/claims.ts): sentences about where
|
|
// weights come from are dropped, then every clause naming an upstream checkpoint, and a description that makes a claim
|
|
// Hanzo's pages do not make is left out whole.
|
|
const NAMED = /\b(?:Qwen|Flash-Next|Tongyi|DeepSeek|MiniMax|Wan-AI|Wan2|granite|Nemotron|Bonsai|unsloth|TRELLIS|JeffreyXiang|YuE|m-a-p)/i;
|
|
const LINEAGE = /repackag|re-?host|fine-?tun\w* (?:of|from)|lineage|related open weights/i;
|
|
const CLAIMS = [
|
|
/SOC ?2 Type ?(II|2)\b/i, /ISO[/ ]?(IEC )?27001/i, /FedRAMP/i, /PCI[- ]DSS/i, /HIPAA (BAA|compliant)/i, /BAA (signed|available|compatible)/i, /\bIL[56]\b/,
|
|
/broker-dealer/i, /transfer[- ]agent/i, /alternative trading system/i, /\bATS, /,
|
|
/\$[\d.,]+ of (prepaid )?credit/i, /\bincludes?\b[^.]{0,60}\bof (prepaid )?credit\b/i, /credit included/i, /included credit/i, /billed at cost from prepaid credit/i,
|
|
/first 24 ?h(ours)?\b/i, /case study( |\s)*↗/i,
|
|
];
|
|
export function said(text) {
|
|
const kept = (text?.trim() ?? '').split(/(?<=[.;])\s+/).filter((s) => !LINEAGE.test(s)).map((s) => {
|
|
if (!NAMED.test(s)) return s;
|
|
const clauses = s.replace(/[.;]$/, '').split(/,\s+/).filter((c) => !NAMED.test(c));
|
|
return clauses.length ? clauses.join(', ') + (/[.;]$/.exec(s)?.[0] ?? '') : '';
|
|
}).filter(Boolean);
|
|
const cut = kept.join(' ').replace(/;$/, '.').trim().replace(/\s*\([^)]*$/, '').trim();
|
|
return cut && !CLAIMS.some((p) => p.test(cut)) ? cut : undefined;
|
|
}
|
|
|
|
// The estate's year, every page of its public repositories. It must answer whole: a partial answer is not a quiet year.
|
|
export async function estate() {
|
|
const page = async (n) => {
|
|
const res = await fetch(`${ACTIVITY}?limit=200&page=${n}`, { headers: { accept: 'application/json' }, signal: AbortSignal.timeout(30000) });
|
|
if (!res.ok) throw new Error(`${res.status} from ${ACTIVITY}`);
|
|
return res.json();
|
|
};
|
|
const first = await page(1);
|
|
if (first?.complete !== true || !first.totals || !Array.isArray(first.groups) || !Array.isArray(first.orgs) || !Array.isArray(first.repos?.data)) throw new Error('unexpected data: the estate activity');
|
|
const rest = await Promise.all(Array.from({ length: Math.max(0, Math.ceil(first.repos.total / 200) - 1) }, (_, i) => page(i + 2)));
|
|
const repos = [first, ...rest].flatMap((p) => p.repos.data);
|
|
if (repos.length !== first.repos.total) throw new Error(`unexpected data: ${repos.length} of the ${first.repos.total} public repositories the estate counts`);
|
|
return { ...first, repos };
|
|
}
|
|
|
|
// Run fn over items, n at a time, keeping the order of the answers.
|
|
async function pool(items, n, fn) {
|
|
const out = new Array(items.length);
|
|
let next = 0;
|
|
await Promise.all(Array.from({ length: Math.min(n, items.length) }, async () => { while (next < items.length) { const i = next++; out[i] = await fn(items[i]); } }));
|
|
return out;
|
|
}
|
|
|
|
// What only GitHub knows. `kept` is the last read this returned; with `thrifty` its cards' contributors and sites and its
|
|
// orgs' descriptions are reused instead of asked again.
|
|
export async function github({ index, token, kept, thrifty }) {
|
|
if (!token) throw new Error('needs GITHUB_TOKEN: the organizations take about 150 requests and GitHub allows 60 an hour without one');
|
|
const head = { accept: 'application/vnd.github+json', authorization: 'Bearer ' + token };
|
|
// a read asked again when the network drops it or GitHub says wait: one slow answer in two thousand must not cost the year
|
|
const call = async (url) => {
|
|
for (let i = 0; ; i++) {
|
|
try {
|
|
const res = await fetch(url, { headers: head, signal: AbortSignal.timeout(20000) });
|
|
if (i < 2 && (res.status === 403 || res.status === 429 || res.status >= 500)) { await res.body?.cancel(); throw new Error(String(res.status)); }
|
|
return res;
|
|
} catch (e) { if (i === 2) throw e; await new Promise((r) => setTimeout(r, 2000 * (i + 1))); }
|
|
}
|
|
};
|
|
const fail = (url, res) => new Error(`${res.status} from ${url}${res.headers.get('x-ratelimit-remaining') === '0' ? ' (rate limited)' : ''}`);
|
|
const last = (link, n) => Number(/[?&]page=(\d+)>; rel="last"/.exec(link ?? '')?.[1] ?? n);
|
|
// one page of an org's public repositories and the number of its last page; null for an org gone from GitHub
|
|
const one = async (org, n) => {
|
|
const url = `${GITHUB}/orgs/${org}/repos?type=public&per_page=100&page=${n}`;
|
|
const res = await call(url);
|
|
if (res.status === 404 && n === 1) return null;
|
|
if (!res.ok) throw fail(url, res);
|
|
const rows = await res.json();
|
|
if (!Array.isArray(rows)) throw new Error('unexpected data: ' + url);
|
|
return { rows, last: last(res.headers.get('link'), n) };
|
|
};
|
|
const list = async (org) => {
|
|
const first = await one(org, 1);
|
|
if (!first) return null;
|
|
return [first, ...(await Promise.all(Array.from({ length: first.last - 1 }, (_, i) => one(org, i + 2))))].flatMap((p) => p.rows);
|
|
};
|
|
// how many people GitHub counts as a repository's contributors, and the first five, bots left out of both
|
|
const people = async (full) => {
|
|
const url = `${GITHUB}/repos/${full}/contributors?per_page=100`;
|
|
const res = await call(url);
|
|
if (!res.ok) throw fail(url, res);
|
|
const humans = (await res.json()).filter((r) => r.login && r.type !== 'Bot');
|
|
const faces = humans.slice(0, 5).map((r) => r.login);
|
|
// a hundred or more is a page to count: ask for one a page and read the last
|
|
if (last(res.headers.get('link'), 1) > 1) {
|
|
const more = await call(url.replace('per_page=100', 'per_page=1'));
|
|
if (!more.ok) throw fail(url, more);
|
|
return { contributors: last(more.headers.get('link'), 1), faces };
|
|
}
|
|
return { contributors: humans.length, faces };
|
|
};
|
|
// a project's own site, kept only when it answers a reader: a card links nothing that does not open
|
|
const site = async (url) => {
|
|
if (!url || !/^https:\/\/[^\s]+$/.test(url.trim())) return undefined;
|
|
try {
|
|
const res = await fetch(url.trim(), { redirect: 'follow', signal: AbortSignal.timeout(10000), headers: { 'user-agent': 'Mozilla/5.0 (compatible; hanzo.ai site)' } });
|
|
return res.ok ? url.trim() : undefined;
|
|
} catch { return undefined; }
|
|
};
|
|
|
|
const indexed = new Map(index.repos.map((r) => [r.repo.toLowerCase(), r]));
|
|
const year = new Map([...indexed].map(([k, r]) => [k, r.commits]));
|
|
const groups = {};
|
|
for (const [group, g] of Object.entries(GROUPS)) {
|
|
const orgs = index.orgs.filter((o) => o.group === group && o.github).map((o) => o.name).sort();
|
|
const held = (await pool(orgs, 8, async (org) => ({ org, rows: await list(org) }))).filter((l) => l.rows);
|
|
const rows = held.flatMap((l) => l.rows);
|
|
const own = rows.filter((r) => !r.fork);
|
|
// the top of a group: the most starred, then forked, then committed to, then the newest
|
|
const rank = (a, b) => (b.stargazers_count ?? 0) - (a.stargazers_count ?? 0) || (b.forks_count ?? 0) - (a.forks_count ?? 0)
|
|
|| (year.get(b.full_name.toLowerCase()) ?? 0) - (year.get(a.full_name.toLowerCase()) ?? 0) || Date.parse(b.pushed_at ?? '0') - Date.parse(a.pushed_at ?? '0');
|
|
const shown = own.filter((r) => !UNLISTED.has(r.full_name));
|
|
const lead = g.featured.map((full) => shown.find((r) => r.full_name === full && !r.archived)).filter(Boolean);
|
|
const picked = [...lead, ...shown.filter((r) => !r.archived && !lead.includes(r)).sort(rank)].slice(0, Math.max(6, g.featured.length));
|
|
const cards = new Map((kept?.groups?.[group]?.top ?? []).map((c) => [c.repo, c]));
|
|
const top = await pool(picked, 6, async (r) => {
|
|
const c = cards.get(r.full_name);
|
|
const [who, home] = thrifty && c?.contributors != null ? [c, c.site] : await Promise.all([people(r.full_name), site(r.homepage)]);
|
|
// a repository public on GitHub but not on the forge is not in the estate's list: its year is the last one read
|
|
const f = indexed.get(r.full_name.toLowerCase());
|
|
return { repo: r.full_name, description: said(r.description), language: r.language ?? undefined, stars: r.stargazers_count ?? 0, forks: r.forks_count ?? 0, commits: f?.commits || c?.commits, months: f?.months ?? c?.months, contributors: who.contributors, faces: who.faces, pushed: r.pushed_at ?? undefined, site: home };
|
|
});
|
|
const tongues = new Map();
|
|
for (const r of own) if (r.language) tongues.set(r.language, (tongues.get(r.language) ?? 0) + 1);
|
|
const languages = [...tongues].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).map(([name, count]) => ({ name, count }));
|
|
const holding = held.filter((l) => l.rows.some((r) => !r.fork));
|
|
const orgsKept = new Map((kept?.groups?.[group]?.orgs ?? []).map((o) => [o.login, o]));
|
|
const about = await pool(holding, 10, async ({ org }) => {
|
|
if (thrifty && orgsKept.has(org)) return orgsKept.get(org);
|
|
const url = `${GITHUB}/orgs/${org}`;
|
|
const res = await call(url);
|
|
if (!res.ok) throw fail(url, res);
|
|
return res.json();
|
|
});
|
|
const directory = holding.map(({ org, rows: mine }, i) => ({
|
|
login: org, name: about[i].name?.trim() || undefined, description: said(about[i].description),
|
|
repos: mine.filter((r) => !r.fork).length, stars: mine.filter((r) => !r.fork).reduce((n, r) => n + (r.stargazers_count ?? 0), 0),
|
|
}));
|
|
groups[group] = {
|
|
checked: held.length, repos: own.length, public: rows.length,
|
|
stars: rows.reduce((n, r) => n + (r.stargazers_count ?? 0), 0),
|
|
forks: rows.reduce((n, r) => n + (r.forks_count ?? 0) + (r.fork ? 1 : 0), 0),
|
|
languages, orgs: directory, top,
|
|
};
|
|
}
|
|
return { source: GITHUB + '/orgs', groups };
|
|
}
|
|
|
|
// The estate and GitHub as sync-oss.ts writes them: `sum` is oss-stats.json (per group the counts, the languages, the
|
|
// organizations and the cards; the estate whole; the forge) and `act` is oss-activity.json (per group the days and the
|
|
// newest commits). A mirror the forge holds counts once among a group's stars and once among its forks.
|
|
export function combine(index, gh) {
|
|
const today = Date.parse(`${index.window.to}T00:00:00Z`);
|
|
const indexed = new Map(index.repos.map((r) => [r.repo.toLowerCase(), r]));
|
|
const forged = new Map(index.orgs.map((o) => [o.name.toLowerCase(), o]));
|
|
const groups = {}, groupsAct = {};
|
|
for (const [group, g] of Object.entries(GROUPS)) {
|
|
const counted = index.groups.find((x) => x.name === group);
|
|
const h = gh.groups[group];
|
|
if (!counted || !h) throw new Error(`${g.name}: the estate's activity or GitHub names no such group`);
|
|
const days = (n) => { let sum = 0; for (let t = today - (n - 1) * DAY; t <= today; t += DAY) sum += counted.daily[new Date(t).toISOString().slice(0, 10)] ?? 0; return sum; };
|
|
groups[group] = {
|
|
name: g.name, lead: g.lead, checked: h.checked,
|
|
orgs: h.orgs.map((o) => { const f = forged.get(o.login.toLowerCase()); return { ...o, commits: f?.commits, months: f?.months, forge: f?.forge }; })
|
|
.sort((a, b) => b.repos - a.repos || a.login.localeCompare(b.login)),
|
|
repos: h.repos, public: h.public, stars: h.stars + counted.mirrors, forks: h.forks + counted.mirrors,
|
|
commits: { year: counted.commits, month: days(30), week: days(7) }, active30: counted.active30,
|
|
estate: Object.fromEntries(['orgs', 'repos', 'public', 'private', 'active30', 'commits', 'merges', 'ours', 'contributors', 'people', 'months', 'mergeMonths', 'oursMonths'].map((k) => [k, counted[k]])),
|
|
languages: h.languages,
|
|
// only the page the estate lists: a forge address made up from a name is a 404 for a repository the forge does not mirror
|
|
top: h.top.map((c) => { const r = indexed.get(c.repo.toLowerCase()); return { ...c, commits: r?.commits || c.commits, months: r?.months ?? c.months, forge: r?.forge }; }),
|
|
};
|
|
groupsAct[group] = {
|
|
daily: counted.daily, active: counted.active,
|
|
latest: counted.latest.slice(0, 8).map((c) => { const home = indexed.get(c.repo.toLowerCase())?.github; return { repo: c.repo, at: c.at, sha: c.sha.slice(0, 7), message: c.message.slice(0, 100), url: home ? `${home}/commit/${c.sha}` : c.url, ours: c.ours }; }),
|
|
};
|
|
}
|
|
const t = index.totals;
|
|
return {
|
|
sum: {
|
|
read: index.read, source: ACTIVITY, groups,
|
|
estate: { read: index.read, window: index.window, months: index.months, complete: index.complete, orgs: t.orgs, repos: t.repos, public: t.public, private: t.private, active: t.active, active30: t.active30, commits: t.commits, merges: t.merges, ours: t.ours, contributors: t.contributors, people: t.people, monthly: t.months, mergeMonths: t.mergeMonths, oursMonths: t.oursMonths },
|
|
// the forge's public figures: what it serves a reader who is signed out
|
|
forge: { read: index.read, source: FORGE, repos: t.public, community: index.repos.filter((r) => r.org === 'hanzo-community').length, ours: index.repos.filter((r) => r.github).length },
|
|
},
|
|
act: { read: index.read, source: ACTIVITY, from: index.window.from, to: index.window.to, groups: groupsAct },
|
|
};
|
|
}
|