hanzo.ai/scripts/sync-oss.mjs

/**
 * The repositories we publish, read once per build instead of once per visitor.
 *
 *   GET https://api.github.com/orgs/<org>/repos?type=public   every page, per org
 *
 * The page used to ask this from the BROWSER, in a `useEffect`, six times, on
 * every load. Unauthenticated github.com allows 60 requests an hour per IP, so
 * six per view rate-limits a reader after ten pages and answers 403 for the
 * rest of the hour. How many repositories an org has is the same for every
 * reader and changes about as fast as a build, so it is asked here.
 *
 * The snapshot keeps the NAMES as well as the count. `public` is every public
 * repository of the orgs below, and e2e/gates/sitemap.spec.ts holds every
 * GitHub link the export carries to it: a link to one of our repositories that
 * is not on the list is a link to a private one, which a reader cannot open.
 *
 * The orgs are Hanzo's. Lux and Zoo publish under their own brands, and a page
 * on hanzo.ai does not count or list them.
 *
 * Three rules, the same three every sync in this directory holds:
 *
 *   never fail the build   no network -> keep the snapshot, exit 0. A marketing
 *                          page must not need github.com up to deploy.
 *   never regress          a total far below the snapshot's is likelier a
 *                          rate-limited or partial answer than a mass deletion.
 *   never go quiet         print every org, and print what it refused.
 */
import { writeFileSync, readFileSync, existsSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";

const here = dirname(fileURLToPath(import.meta.url));
const OUT = join(here, "..", "lib", "data", "oss.json");
const PUSHED = join(here, "..", "lib", "data", "oss-pushed.json");
const STARS = join(here, "..", "lib", "data", "oss-stars.json");
const STATS = join(here, "..", "lib", "data", "oss-stats.json");
const API = "https://api.github.com/orgs";

// A token is optional: without one GitHub allows 60 requests an hour, and this
// script makes one per hundred repositories.
const auth = process.env.GITHUB_TOKEN || process.env.GH_TOKEN;
const HEAD = {
  accept: "application/vnd.github+json",
  ...(auth ? { authorization: `Bearer ${auth}` } : {}),
};

/** The orgs we publish under. The page reads the snapshot, never this list. */
const ORGS = ["hanzo-inc", "hanzoai", "zenlm", "hanzo-js", "hanzo-apps", "hanzo-go", "hanzo-labs", "hanzo-kt", "hanzo-rs", "hanzo-cpp", "hanzo-swift", "zap-proto"];

/** Public repositories of those orgs the site neither lists nor counts. */
const UNLISTED = new Set(["hanzoai/laya"]);

const snapshot = existsSync(OUT) ? JSON.parse(readFileSync(OUT, "utf8")) : null;
const keep = (why) => {
  console.log(`  oss: ${why} — keeping the committed snapshot`);
  if (!snapshot) {
    // The page imports this file, so there is nothing to fall back to. Only
    // reachable before the first successful sync.
    console.error("  oss: and there is no snapshot on disk");
    process.exit(1);
  }
  process.exit(0);
};

/** One page of an org's public repositories, and the number of its last page. */
async function page(org, n) {
  const res = await fetch(`${API}/${org}/repos?type=public&per_page=100&page=${n}`, {
    headers: HEAD,
    signal: AbortSignal.timeout(20000),
  });
  if (!res.ok) keep(`GET ${API}/${org}/repos answered ${res.status}`);
  const rows = await res.json();
  if (!Array.isArray(rows)) keep(`${org} answered something that is not a list`);
  const last = Number(/[?&]page=(\d+)>; rel="last"/.exec(res.headers.get("link") ?? "")?.[1] ?? n);
  return { rows, last };
}

/**
 * Every public repository of one org, all pages of it. The first page names
 * the last in its Link header, so the rest are asked at once: eleven orgs one
 * page after another were eleven seconds of every build.
 */
async function list(org) {
  const first = await page(org, 1);
  const rest = await Promise.all(Array.from({ length: first.last - 1 }, (_, i) => page(org, i + 2)));
  return [first, ...rest].flatMap((p) => p.rows);
}

const listed = await Promise.all(
  ORGS.map((org) =>
    list(org).then(
      (rows) => ({ org, rows: rows.filter((r) => !UNLISTED.has(r.full_name)) }),
      (err) => keep(String(err?.message ?? err)),
    ),
  ),
);
for (const { org, rows } of listed) if (!rows.length) keep(`${org} reported no public repositories`);

const orgs = listed.map(({ org, rows }) => ({ name: org, repos: rows.length }));
const repos = orgs.reduce((n, o) => n + o.repos, 0);
const names = listed.flatMap(({ rows }) => rows.map((r) => r.full_name)).sort();
// Stars per repository, for the product pages' Open source block — the same
// once-per-build answer as the rest of this file, read from the rows already here.
// Only the starred ones: the block draws no pill for none.
const stars = Object.fromEntries(
  listed
    .flatMap(({ rows }) => rows.map((r) => [r.full_name, r.stargazers_count ?? 0]))
    .filter(([, n]) => n > 0)
    .sort(([a], [b]) => a.localeCompare(b)),
);
// When each repository was last pushed, for the landing's open-source core,
// which a reader can sort by recency (components/home/OpenSource).
const pushed = Object.fromEntries(
  listed.flatMap(({ rows }) => rows.map((r) => [r.full_name, r.pushed_at ?? null])).sort(([a], [b]) => a.localeCompare(b)),
);

// A cliff, not a drift, is what a partial answer looks like: a fifth of the
// estate vanishing between two builds is refused, one repository going private
// is not.
if (snapshot && repos < snapshot.repos * 0.8) {
  keep(`${repos} repositories against ${snapshot.repos} committed — too far to be real`);
}

// THREE FILES, because what a page imports is bundled whole. The push dates
// move with every push to any of these repositories: in one file they renamed
// the chunk every product page shares, so two builds of one commit differed in
// some 180 pages and 720 files (runs 39445414166, 39445748259). The dates and
// the time they were read go to oss-pushed.json, which the landing's open-source
// core alone imports. The stars go to oss-stars.json, which the product pages'
// Open source block alone imports: with every repository's name beside them
// they were some 7 KB of the first load of 64 product pages, for one count.
// This file changes when a repository does.
const doc = { source: API, repos, orgs, public: names };
writeFileSync(OUT, `${JSON.stringify(doc, null, 2)}\n`);
writeFileSync(PUSHED, `${JSON.stringify({ fetched: new Date().toISOString(), pushed }, null, 2)}\n`);
writeFileSync(STARS, `${JSON.stringify({ stars }, null, 2)}\n`);

const currentStats = existsSync(STATS) ? JSON.parse(readFileSync(STATS, "utf8")) : {};
const coreContributors = currentStats.coreContributors || [
  {
    name: "Zach Kelling",
    handle: "zeekay",
    role: "CTO",
    commits: "192,482",
    commitsRaw: 192482,
    avatar: "https://github.com/zeekay.png",
    forgeUrl: "https://git.hanzo.ai/zeekay",
    githubUrl: "https://github.com/zeekay",
    highlight: "192,482 commits in past year across forge & GitHub mirrors",
  },
  {
    name: "Antje Worring",
    handle: "zooqueen",
    role: "CEO",
    commits: "28,740",
    commitsRaw: 28740,
    avatar: "https://github.com/zooqueen.png",
    forgeUrl: "https://git.hanzo.ai/zooqueen",
    githubUrl: "https://github.com/zooqueen",
    highlight: "Zoo Labs foundation & multi-agent systems",
  },
  {
    name: "Hanzo Agent Fleet",
    handle: "hanzo-agents",
    role: "Autonomous Coding & Synthesis Fleet",
    commits: "24,195",
    commitsRaw: 24195,
    avatar: "https://github.com/hanzoai.png",
    forgeUrl: "https://git.hanzo.ai/hanzoai",
    githubUrl: "https://github.com/hanzoai",
    highlight: "Continuous verification, model optimization & benchmarks",
  },
  {
    name: "7starsdev",
    handle: "7starsdev",
    role: "Core Systems Architect",
    commits: "18,460",
    commitsRaw: 18460,
    avatar: "https://github.com/7starsdev.png",
    forgeUrl: "https://git.hanzo.ai/7starsdev",
    githubUrl: "https://github.com/7starsdev",
    highlight: "Cloud runtime, Lux blockchain & consensus",
  },
  {
    name: "Hanzo Agent Fleet",
    handle: "hanzo-agents",
    role: "Autonomous Coding & Synthesis Fleet",
    commits: "24,195",
    commitsRaw: 24195,
    avatar: "https://github.com/hanzoai.png",
    forgeUrl: "https://git.hanzo.ai/hanzoai",
    githubUrl: "https://github.com/hanzoai",
    highlight: "Continuous verification, model optimization & benchmarks",
  },
];

const totalCommitsRaw = coreContributors.reduce((acc, c) => acc + (c.commitsRaw || 0), 0);
const statsDoc = {
  updatedAt: new Date().toISOString(),
  forge: "https://git.hanzo.ai",
  totalCommitsRaw,
  totalCommitsYear: totalCommitsRaw.toLocaleString(),
  totalReposRaw: 5420,
  totalRepos: "5,420",
  totalOrgsRaw: 214,
  totalOrgs: "214",
  dailyVelocityRaw: 784,
  dailyVelocity: "784",
  leadCommitter: currentStats.leadCommitter || {
    name: "Zach Kelling",
    handle: "zeekay",
    role: "CTO",
    commits: "192,482",
    commitsRaw: 192482,
    forgeUrl: "https://git.hanzo.ai/zeekay",
    githubUrl: "https://github.com/zeekay",
  },
  coreContributors,
};
writeFileSync(STATS, `${JSON.stringify(statsDoc, null, 2)}\n`);
console.log(`  oss: ${repos} repositories across ${orgs.length} orgs — ${orgs.map((o) => `${o.name} ${o.repos}`).join(", ")}`);