hanzo.ai/scripts/sync-oss.ts

/**
 * The repositories we publish, read once per build instead of once per visitor.
 *
 *   GET https://api.github.com/orgs/<org>/repos?type=public   every page, per org
 *
 * The page used to ask this from the BROWSER, in a `useEffect`, six times, on
 * every load. Unauthenticated github.com allows 60 requests an hour per IP, so
 * six per view rate-limits a reader after ten pages and answers 403 for the
 * rest of the hour. How many repositories an org has is the same for every
 * reader and changes about as fast as a build, so it is asked here.
 *
 * The snapshot keeps the NAMES as well as the count. `public` is every public
 * repository of the orgs below, and e2e/gates/sitemap.spec.ts holds every
 * GitHub link the export carries to it: a link to one of our repositories that
 * is not on the list is a link to a private one, which a reader cannot open.
 *
 * The orgs are Hanzo's. Lux and Zoo publish under their own brands, and a page
 * on hanzo.ai does not count or list them.
 *
 * Three rules, the same three every sync in this directory holds:
 *
 *   never fail the build   no network -> keep the snapshot, exit 0. A marketing
 *                          page must not need github.com up to deploy.
 *   never regress          a total far below the snapshot's is likelier a
 *                          rate-limited or partial answer than a mass deletion.
 *   never go quiet         print every org, and print what it refused.
 */
import { writeFileSync, readFileSync, existsSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";

const here = dirname(fileURLToPath(import.meta.url));
const OUT = join(here, "..", "lib", "data", "oss.json");
const PUSHED = join(here, "..", "lib", "data", "oss-pushed.json");
const STARS = join(here, "..", "lib", "data", "oss-stars.json");
const STATS = join(here, "..", "lib", "data", "oss-stats.json");
const API = "https://api.github.com/orgs";

// A token is optional: without one GitHub allows 60 requests an hour, and this
// script makes one per hundred repositories.
const auth = process.env.GITHUB_TOKEN || process.env.GH_TOKEN;
/** A repository as GitHub's list answers it, with the fields this file reads. */
type Repo = { full_name: string; fork?: boolean; archived?: boolean; stargazers_count?: number; pushed_at?: string | null }
type Week = { week: number; days: number[] }
type Commit = { repo: string; at: string; sha: string; message: string }

const HEAD: Record<string, string> = {
  accept: "application/vnd.github+json",
  ...(auth ? { authorization: `Bearer ${auth}` } : {}),
};

/** The orgs we publish under. The page reads the snapshot, never this list. */
const ORGS: string[] = JSON.parse(readFileSync(join(here, "..", "lib", "data", "oss-organizations.json"), "utf8"));

/** Implementation repositories counted in aggregates but not published as links. */
const UNLISTED = new Set(["hanzoai/laya"]);

const snapshot = existsSync(OUT) ? JSON.parse(readFileSync(OUT, "utf8")) : null;
const keep = (why: string): never => {
  console.log(`  oss: ${why} — keeping the committed snapshot`);
  if (!snapshot) {
    // The page imports this file, so there is nothing to fall back to. Only
    // reachable before the first successful sync.
    console.error("  oss: and there is no snapshot on disk");
    process.exit(1);
  }
  process.exit(0);
};

/** One page of an org's public repositories, and the number of its last page. */
async function page(org: string, n: number): Promise<{ rows: Repo[]; last: number }> {
  const res = await fetch(`${API}/${org}/repos?type=public&per_page=100&page=${n}`, {
    headers: HEAD,
    signal: AbortSignal.timeout(20000),
  });
  if (!res.ok) keep(`GET ${API}/${org}/repos answered ${res.status}`);
  const rows = await res.json();
  if (!Array.isArray(rows)) keep(`${org} answered something that is not a list`);
  const last = Number(/[?&]page=(\d+)>; rel="last"/.exec(res.headers.get("link") ?? "")?.[1] ?? n);
  return { rows, last };
}

/**
 * Every public repository of one org, all pages of it. The first page names
 * the last in its Link header, so the rest are asked at once: eleven orgs one
 * page after another were eleven seconds of every build.
 */
async function list(org: string): Promise<Repo[]> {
  const first = await page(org, 1);
  const rest = await Promise.all(Array.from({ length: first.last - 1 }, (_, i) => page(org, i + 2)));
  return [first, ...rest].flatMap((p) => p.rows);
}

const listed = await Promise.all(
  ORGS.map((org) =>
    list(org).then(
      (rows) => ({ org, rows }),
      (err) => keep(String(err?.message ?? err)),
    ),
  ),
);
for (const { org, rows } of listed) {
  const previous = snapshot?.orgs?.find((o: { name: string; repos: number }) => o.name === org);
  if (!rows.length && previous?.repos > 0) keep(`${org} reported no public repositories against ${previous.repos} committed`);
}

const orgs = listed.map(({ org, rows }) => ({ name: org, repos: rows.length }));
const repos = orgs.reduce((n, o) => n + o.repos, 0);
const visible = listed.flatMap(({ rows }) => rows).filter((r) => !UNLISTED.has(r.full_name));
const names = visible.map((r) => r.full_name).sort();
// Stars per repository, for the product pages' Open source block — the same
// once-per-build answer as the rest of this file, read from the rows already here.
// Only the starred ones: the block draws no pill for none.
const stars = Object.fromEntries(
  visible
    .map((r): [string, number] => [r.full_name, r.stargazers_count ?? 0])
    .filter(([, n]) => n > 0)
    .sort(([a], [b]) => a.localeCompare(b)),
);
// When each repository was last pushed, for the landing's open-source core,
// which a reader can sort by recency (components/home/OpenSource).
const pushed = Object.fromEntries(
  visible.map((r): [string, string | null] => [r.full_name, r.pushed_at ?? null]).sort(([a], [b]) => a.localeCompare(b)),
);

// A cliff, not a drift, is what a partial answer looks like: a fifth of the
// estate vanishing between two builds is refused, one repository going private
// is not.
if (snapshot && repos < snapshot.repos * 0.8) {
  keep(`${repos} repositories against ${snapshot.repos} committed — too far to be real`);
}

// THREE FILES, because what a page imports is bundled whole. The push dates
// move with every push to any of these repositories: in one file they renamed
// the chunk every product page shares, so two builds of one commit differed in
// some 180 pages and 720 files (runs 39445414166, 39445748259). The dates and
// the time they were read go to oss-pushed.json, which the landing's open-source
// core alone imports. The stars go to oss-stars.json, which the product pages'
// Open source block alone imports: with every repository's name beside them
// they were some 7 KB of the first load of 64 product pages, for one count.
// This file changes when a repository does.
// `repos` is every public repository (forks included: it is what the sitemap's links
// are held to and what the cliff check compares); `originals` is the repositories we
// wrote, forks left out, and is the one number the site states about its size.
const originals = listed.flatMap(({ rows }) => rows).filter((r) => !r.fork).length;
const doc = { source: API, repos, originals, orgs, public: names };
writeFileSync(OUT, `${JSON.stringify(doc, null, 2)}\n`);
// Every surface quoting repository totals follows this same inventory.
const countsFile = join(here, "..", "lib", "counts.ts");
const counts = readFileSync(countsFile, "utf8").replace(/export const REPO_COUNT = \d+/, `export const REPO_COUNT = ${originals}`).replace(/export const REPO_FLOOR = \d+/, `export const REPO_FLOOR = ${Math.floor(originals / 100) * 100}`);
writeFileSync(countsFile, counts);
writeFileSync(PUSHED, `${JSON.stringify({ fetched: new Date().toISOString(), pushed }, null, 2)}\n`);
writeFileSync(STARS, `${JSON.stringify({ stars }, null, 2)}\n`);

console.log(`  oss: ${repos} repositories across ${orgs.length} orgs — ${orgs.map((o) => `${o.name} ${o.repos}`).join(", ")}`);

// ACTIVITY. What the landing says about shipping is counted here and nowhere
// else: every number below is read off GitHub's public API for the repositories
// listed above, never typed. Forks are left out because their history is the
// upstream's, and a fork's commit count is not ours; archived repositories stay in,
// so the repository count here is the `originals` one in oss.json.
//
//   GET /repos/<repo>/stats/commit_activity   52 weeks of daily commit counts
//   GET /repos/<repo>/commits?per_page=1       the newest commit
//
// A repository whose weekly counts GitHub has not computed yet answers 202; it is
// asked again, and one still not ready is counted out and said so (`counted`
// against `of`), so a total is never presented as whole when it is not.
const ACTIVITY = join(here, "..", "lib", "data", "oss-activity.json");
const REPOS_API = "https://api.github.com/repos";
const DAY = 86400;
const sleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms));

/** The day's key, UTC. */
const key = (sec: number) => new Date(sec * 1000).toISOString().slice(0, 10);

// An archived repository is still ours and its commits in the year are ours: only forks are left out.
const own = listed.flatMap(({ rows }) => rows).filter((r) => !r.fork);
const starred = own.reduce((n, r) => n + (r.stargazers_count ?? 0), 0);

async function weekly(full: string): Promise<Week[] | null> {
  for (let i = 0; i < 3; i++) {
    const res = await fetch(`${REPOS_API}/${full}/stats/commit_activity`, { headers: HEAD, signal: AbortSignal.timeout(20000) });
    if (res.status === 200) {
      const rows = await res.json();
      return Array.isArray(rows) ? rows : [];
    }
    if (res.status === 202) {
      await sleep(2500);
      continue;
    }
    if (res.status === 403 || res.status === 429) throw new Error(`${full}: ${res.status}, rate limited`);
    return res.status === 204 || res.status === 404 || res.status === 409 ? [] : null;
  }
  return null;
}

async function newest(full: string): Promise<Commit | null> {
  const res = await fetch(`${REPOS_API}/${full}/commits?per_page=1`, { headers: HEAD, signal: AbortSignal.timeout(20000) });
  if (!res.ok) return null;
  const [c] = await res.json();
  const at = c?.commit?.committer?.date;
  if (!c || !at) return null;
  return { repo: full, at, sha: String(c.sha).slice(0, 7), message: String(c.commit.message).split("\n")[0].slice(0, 100) };
}

/** Run `fn` over `items`, `n` at a time, keeping the order of the answers. */
async function pool<T, R>(items: T[], n: number, fn: (item: T) => Promise<R>): Promise<R[]> {
  const out: R[] = new Array(items.length);
  let next = 0;
  await Promise.all(
    Array.from({ length: n }, async () => {
      while (next < items.length) {
        const i = next++;
        out[i] = await fn(items[i]);
      }
    }),
  );
  return out;
}

try {
  const today = Math.floor(Date.now() / 1000 / DAY) * DAY;
  const from = today - 364 * DAY;
  // Only a repository pushed in the window can hold a commit in it.
  const live = own.filter((r) => r.pushed_at && Date.parse(r.pushed_at) / 1000 >= from);
  const series = await pool(live, 10, (r) => weekly(r.full_name));
  const daily: Record<string, number> = {};
  const active: Record<string, number> = {};
  let counted = 0;
  let month = 0;
  series.forEach((weeks) => {
    if (!weeks) return;
    counted++;
    let recent = false;
    for (const w of weeks) {
      w.days.forEach((n, d) => {
        const t = w.week + d * DAY;
        if (t < from || t > today || n <= 0) return;
        daily[key(t)] = (daily[key(t)] ?? 0) + n;
        active[key(t)] = (active[key(t)] ?? 0) + 1;
        if (t > today - 30 * DAY) recent = true;
      });
    }
    if (recent) month++;
  });
  const sum = (days: number) => {
    let n = 0;
    for (let t = today - (days - 1) * DAY; t <= today; t += DAY) n += daily[key(t)] ?? 0;
    return n;
  };
  const year = sum(365);

  // A cliff is a partial answer, not a quiet year.
  const before = existsSync(STATS) ? JSON.parse(readFileSync(STATS, "utf8")) : null;
  if (before?.commits?.year && year < before.commits.year * 0.5) {
    throw new Error(`${year} commits against ${before.commits.year} committed — too far to be real`);
  }
  if (counted < live.length * 0.8) throw new Error(`only ${counted} of ${live.length} repositories answered`);

  const recent = own
    .filter((r) => r.pushed_at && !UNLISTED.has(r.full_name))
    .sort((a, b) => Date.parse(b.pushed_at) - Date.parse(a.pushed_at))
    .slice(0, 24);
  const latest = (await pool(recent, 8, (r) => newest(r.full_name)))
    .filter(Boolean)
    .sort((a, b) => Date.parse(b.at) - Date.parse(a.at))
    .slice(0, 8);

  const read = new Date().toISOString();
  writeFileSync(ACTIVITY, `${JSON.stringify({ read, source: "api.github.com", from: key(from), to: key(today), daily, active, latest }, null, 2)}\n`);
  const stats = {
    read,
    source: "api.github.com",
    orgs: orgs.length,
    repos: own.length,
    stars: starred,
    commits: { year, month: sum(30), week: sum(7) },
    active30: month,
    counted,
    of: live.length,
  };
  writeFileSync(STATS, `${JSON.stringify(stats, null, 2)}\n`);
  console.log(`  oss: ${year} commits in a year, ${stats.commits.week} this week, from ${counted} of ${live.length} repositories pushed in it`);
} catch (err) {
  console.log(`  oss: activity not read (${String(err?.message ?? err)}) — keeping the committed snapshot`);
}