hanzo.ai/scripts/sync-oss.ts

/**
 * The repositories we publish, read once per build instead of once per visitor.
 *
 *   GET https://api.github.com/orgs/<org>/repos?type=public   every page, per org
 *
 * The page used to ask this from the BROWSER, in a `useEffect`, six times, on
 * every load. Unauthenticated github.com allows 60 requests an hour per IP, so
 * six per view rate-limits a reader after ten pages and answers 403 for the
 * rest of the hour. How many repositories an org has is the same for every
 * reader and changes about as fast as a build, so it is asked here.
 *
 * The snapshot keeps the NAMES as well as the count. `public` is every public
 * repository of the orgs below, and e2e/gates/sitemap.spec.ts holds every
 * GitHub link the export carries to it: a link to one of our repositories that
 * is not on the list is a link to a private one, which a reader cannot open.
 *
 * The accounts are the work, not one brand. Hanzo, Lux, Zoo, Zen, Zap, Enso
 * and Sensei, plus the public repositories github.com/zeekay owns. A side
 * project that is none of those is not in this list. The forge at
 * forge.hanzo.ai is a different host; this script reads GitHub, which is the
 * count the site can check.
 *
 * Three rules, the same three every sync in this directory holds:
 *
 *   never fail the build   no network -> keep the snapshot, exit 0. A marketing
 *                          page must not need github.com up to deploy.
 *   never regress          a total far below the snapshot's is likelier a
 *                          rate-limited or partial answer than a mass deletion.
 *   never go quiet         print every org, and print what it refused.
 */
import { writeFileSync, readFileSync, existsSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { privateDaily } from "./private-days.mjs";

const here = dirname(fileURLToPath(import.meta.url));
const OUT = join(here, "..", "lib", "data", "oss.json");
const PUSHED = join(here, "..", "lib", "data", "oss-pushed.json");
const STARS = join(here, "..", "lib", "data", "oss-stars.json");
const STATS = join(here, "..", "lib", "data", "oss-stats.json");

// A token is optional: without one GitHub allows 60 requests an hour, and this
// script makes one per hundred repositories.
const auth = process.env.GITHUB_TOKEN || process.env.GH_TOKEN;
/** A repository as GitHub's list answers it, with the fields this file reads. */
type Repo = { full_name: string; private?: boolean; fork?: boolean; archived?: boolean; stargazers_count?: number; pushed_at?: string | null }
type Week = { week: number; days: number[] }
type Commit = { repo: string; at: string; sha: string; message: string }

const HEAD: Record<string, string> = {
  accept: "application/vnd.github+json",
  ...(auth ? { authorization: `Bearer ${auth}` } : {}),
};

/**
 * The orgs we publish under. Every one has at least one public repository;
 * an empty org makes the sync keep the previous snapshot. The page reads the
 * snapshot, never this list.
 */
const ORGS = [
  "ellipsis",
  "ellipsis-index",
  "hanzo-agency",
  "hanzo-app",
  "hanzo-ext",
  "hanzo-industries",
  "hanzo-lm",
  "hanzo-ml",
  "hanzoapp",
  "hanzocr",
  "hanzodao",
  "hanzodb",
  "hanzodns",
  "hanzodocs",
  "hanzofn",
  "hanzogui",
  "hanzohq",
  "hanzoid",
  "hanzoio",
  "hanzolm",
  "hanzolog",
  "hanzometric",
  "hanzomsg",
  "hanzonetwork",
  "hanzoo11y",
  "hanzoskills",
  "hanzosql",
  "hanzovm",
  "hanzoworks",
  "hanzo-apps",
  "hanzo-core",
  "hanzo-cpp",
  "hanzo-go",
  "hanzo-inc",
  "hanzo-io",
  "hanzo-js",
  "hanzo-kt",
  "hanzo-labs",
  "hanzo-rs",
  "hanzo-studio",
  "hanzo-swift",
  "hanzoai",
  "hanzobot",
  "hanzokms",
  "hanzokv",
  "hanzonet",
  "hanzos3",
  "hanzoskill",
  "hanzoui",
  "hanzozt",
  "lux-apps",
  "lux-js",
  "luxdao",
  "luxdefi",
  "luxexchange",
  "luxfhe",
  "luxfi",
  "luxstatic",
  "sakejs",
  "sensei-group",
  "shopjs",
  "zap-proto",
  "zenlm",
  "zoo-apps",
  "zoo-labs",
  "zooai",
];

/** People whose own public repositories are part of the same count. Their forks of other people's repositories are not: that history is not ours. An org's fork is, because that is a project we track. */
const USERS = ["zeekay"];

/** GitHub's own contribution count for the past year, the number on the profile. */
const PEOPLE = ["zeekay", "zooqueen"];

/** Public repositories of those orgs the site neither lists nor counts. */
const UNLISTED = new Set(["hanzoai/laya"]);

const snapshot = existsSync(OUT) ? JSON.parse(readFileSync(OUT, "utf8")) : null;
const keep = (why: string): never => {
  console.log(`  oss: ${why} — keeping the committed snapshot`);
  if (!snapshot) {
    // The page imports this file, so there is nothing to fall back to. Only
    // reachable before the first successful sync.
    console.error("  oss: and there is no snapshot on disk");
    process.exit(1);
  }
  process.exit(0);
};

/** One page of an account's public repositories, and the number of its last page. */
async function page(account: string, kind: "orgs" | "users", n: number): Promise<{ rows: Repo[]; last: number }> {
  const type = kind === "orgs" ? "public" : "owner";
  const url = `https://api.github.com/${kind}/${account}/repos?type=${type}&per_page=100&page=${n}`;
  const res = await fetch(url, { headers: HEAD, signal: AbortSignal.timeout(20000) });
  if (!res.ok) keep(`GET ${kind}/${account}/repos answered ${res.status}`);
  const rows = await res.json();
  if (!Array.isArray(rows)) keep(`${account} answered something that is not a list`);
  const last = Number(/[?&]page=(\d+)>; rel="last"/.exec(res.headers.get("link") ?? "")?.[1] ?? n);
  return { rows, last };
}

/**
 * Every public repository of one org, all pages of it. The first page names
 * the last in its Link header, so the rest are asked at once: eleven orgs one
 * page after another were eleven seconds of every build.
 */
async function list(account: string, kind: "orgs" | "users"): Promise<Repo[]> {
  const first = await page(account, kind, 1);
  const rest = await Promise.all(Array.from({ length: first.last - 1 }, (_, i) => page(account, kind, i + 2)));
  return [first, ...rest].flatMap((p) => p.rows);
}

const accounts = [
  ...ORGS.map((name) => ({ name, kind: "orgs" as const })),
  ...USERS.map((name) => ({ name, kind: "users" as const })),
];
const listed = await Promise.all(
  accounts.map(({ name, kind }) =>
    list(name, kind).then(
      (rows) => ({ org: name, rows: rows.filter((r) => !r.private && !UNLISTED.has(r.full_name)) }),
      (err) => keep(String(err?.message ?? err)),
    ),
  ),
);
const empty = listed.filter(({ rows }) => !rows.length).map(({ org }) => org);
if (empty.length) console.log(`  oss: no public repositories — ${empty.join(", ")}`);
const present = listed.filter(({ rows }) => rows.length);
if (!present.length) keep("no account reported a public repository");

const orgs = present.map(({ org, rows }) => ({ name: org, repos: rows.length }));
const repos = orgs.reduce((n, o) => n + o.repos, 0);
const names = present.flatMap(({ rows }) => rows.map((r) => r.full_name)).sort();
// Stars per repository, for the product pages' Open source block — the same
// once-per-build answer as the rest of this file, read from the rows already here.
// Only the starred ones: the block draws no pill for none.
const stars = Object.fromEntries(
  present
    .flatMap(({ rows }) => rows.map((r): [string, number] => [r.full_name, r.stargazers_count ?? 0]))
    .filter(([, n]) => n > 0)
    .sort(([a], [b]) => a.localeCompare(b)),
);
// When each repository was last pushed, for the landing's open-source core,
// which a reader can sort by recency (components/home/OpenSource).
const pushed = Object.fromEntries(
  present.flatMap(({ rows }) => rows.map((r): [string, string | null] => [r.full_name, r.pushed_at ?? null])).sort(([a], [b]) => a.localeCompare(b)),
);

// A cliff, not a drift, is what a partial answer looks like: a fifth of the
// estate vanishing between two builds is refused, one repository going private
// is not.
if (snapshot && repos < snapshot.repos * 0.8) {
  keep(`${repos} repositories against ${snapshot.repos} committed — too far to be real`);
}

// THREE FILES, because what a page imports is bundled whole. The push dates
// move with every push to any of these repositories: in one file they renamed
// the chunk every product page shares, so two builds of one commit differed in
// some 180 pages and 720 files (runs 39445414166, 39445748259). The dates and
// the time they were read go to oss-pushed.json, which the landing's open-source
// core alone imports. The stars go to oss-stars.json, which the product pages'
// Open source block alone imports: with every repository's name beside them
// they were some 7 KB of the first load of 64 product pages, for one count.
// This file changes when a repository does.
const doc = { source: "https://api.github.com", repos, orgs, asked: accounts.map((a) => a.name), public: names };
writeFileSync(OUT, `${JSON.stringify(doc, null, 2)}\n`);
writeFileSync(PUSHED, `${JSON.stringify({ fetched: new Date().toISOString(), pushed }, null, 2)}\n`);
writeFileSync(STARS, `${JSON.stringify({ stars }, null, 2)}\n`);

console.log(`  oss: ${repos} repositories across ${orgs.length} orgs — ${orgs.map((o) => `${o.name} ${o.repos}`).join(", ")}`);

// ACTIVITY. What the landing says about shipping is counted here and nowhere
// else: every number below is read off GitHub's public API for the repositories
// listed above, never typed. An org's fork is included, because a tracked fork
// (OpenClaw and the rest) is a project whose main we follow. A person's own
// fork of someone else's repository is not.
//
//   GET /repos/<repo>/stats/commit_activity   52 weeks of daily commit counts
//   GET /repos/<repo>/commits?per_page=1       the newest commit
//
// A repository whose weekly counts GitHub has not computed yet answers 202; it is
// asked again, and one still not ready is counted out and said so (`counted`
// against `of`), so a total is never presented as whole when it is not.
const ACTIVITY = join(here, "..", "lib", "data", "oss-activity.json");
const REPOS_API = "https://api.github.com/repos";
const DAY = 86400;
const sleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms));

/** The day's key, UTC. */
const key = (sec: number) => new Date(sec * 1000).toISOString().slice(0, 10);

const user = new Set(USERS);
const own = present.flatMap(({ org, rows }) => rows.filter((r) => !r.archived && !(user.has(org) && r.fork)));
const starred = own.reduce((n, r) => n + (r.stargazers_count ?? 0), 0);

async function weekly(full: string): Promise<Week[] | null> {
  for (let i = 0; i < 6; i++) {
    const res = await fetch(`${REPOS_API}/${full}/stats/commit_activity`, { headers: HEAD, signal: AbortSignal.timeout(20000) });
    if (res.status === 200) {
      const rows = await res.json();
      return Array.isArray(rows) ? rows : [];
    }
    if (res.status === 202) {
      await sleep(2000);
      continue;
    }
    // One repository waiting out a limit must not throw away the whole read.
    if (res.status === 403 || res.status === 429) {
      const wait = Number(res.headers.get("retry-after")) || 8;
      await sleep(Math.min(wait, 30) * 1000);
      continue;
    }
    return res.status === 204 || res.status === 404 || res.status === 409 ? [] : null;
  }
  return null;
}

async function newest(full: string): Promise<Commit | null> {
  const res = await fetch(`${REPOS_API}/${full}/commits?per_page=1`, { headers: HEAD, signal: AbortSignal.timeout(20000) });
  if (!res.ok) return null;
  const [c] = await res.json();
  const at = c?.commit?.committer?.date;
  if (!c || !at) return null;
  return { repo: full, at, sha: String(c.sha).slice(0, 7), message: String(c.commit.message).split("\n")[0].slice(0, 100) };
}

/** Run `fn` over `items`, `n` at a time, keeping the order of the answers. */
async function pool<T, R>(items: T[], n: number, fn: (item: T) => Promise<R>): Promise<R[]> {
  const out: R[] = new Array(items.length);
  let next = 0;
  await Promise.all(
    Array.from({ length: n }, async () => {
      while (next < items.length) {
        const i = next++;
        out[i] = await fn(items[i]);
      }
    }),
  );
  return out;
}

try {
  const today = Math.floor(Date.now() / 1000 / DAY) * DAY;
  const from = today - 364 * DAY;
  // Only a repository pushed in the window can hold a commit in it.
  const live = own.filter((r) => r.pushed_at && Date.parse(r.pushed_at) / 1000 >= from);
  let done = 0;
  const series = await pool(live, 8, async (r) => {
    const weeks = await weekly(r.full_name);
    done++;
    if (done % 200 === 0) console.log(`  oss: activity ${done} of ${live.length}`);
    return weeks;
  });
  const daily: Record<string, number> = {};
  const active: Record<string, number> = {};
  let counted = 0;
  let month = 0;
  series.forEach((weeks) => {
    if (!weeks) return;
    counted++;
    let recent = false;
    for (const w of weeks) {
      w.days.forEach((n, d) => {
        const t = w.week + d * DAY;
        if (t < from || t > today || n <= 0) return;
        daily[key(t)] = (daily[key(t)] ?? 0) + n;
        active[key(t)] = (active[key(t)] ?? 0) + 1;
        if (t > today - 30 * DAY) recent = true;
      });
    }
    if (recent) month++;
  });
  const priv = await privateDaily(auth, key(from), key(today));
  for (const [day, n] of Object.entries(priv.added)) daily[day] = (daily[day] ?? 0) + n;

  const sum = (days: number) => {
    let n = 0;
    for (let t = today - (days - 1) * DAY; t <= today; t += DAY) n += daily[key(t)] ?? 0;
    return n;
  };
  const year = sum(365);

  // A cliff is a partial answer, not a quiet year.
  const before = existsSync(STATS) ? JSON.parse(readFileSync(STATS, "utf8")) : null;
  if (before?.commits?.year && year < before.commits.year * 0.5) {
    throw new Error(`${year} commits against ${before.commits.year} committed — too far to be real`);
  }
  if (counted < live.length * 0.8) throw new Error(`only ${counted} of ${live.length} repositories answered`);

  const recent = own
    .filter((r) => r.pushed_at)
    .sort((a, b) => Date.parse(b.pushed_at) - Date.parse(a.pushed_at))
    .slice(0, 24);
  const latest = (await pool(recent, 8, (r) => newest(r.full_name)))
    .filter(Boolean)
    .sort((a, b) => Date.parse(b.at) - Date.parse(a.at))
    .slice(0, 8);

  const contributors = (
    await Promise.all(PEOPLE.map(async (login) => {
      if (!auth) return null;
      const res = await fetch("https://api.github.com/graphql", {
        method: "POST",
        headers: { ...HEAD, "content-type": "application/json" },
        body: JSON.stringify({
          query: "query($login:String!){ user(login:$login){ contributionsCollection { contributionCalendar { totalContributions } restrictedContributionsCount } } }",
          variables: { login },
        }),
        signal: AbortSignal.timeout(20000),
      });
      if (!res.ok) return null;
      const body = await res.json();
      const box = body?.data?.user?.contributionsCollection;
      const contributions = box?.contributionCalendar?.totalContributions;
      if (typeof contributions !== "number") return null;
      return { login, contributions, restricted: box.restrictedContributionsCount ?? 0 };
    }))
  ).filter(Boolean);

  const read = new Date().toISOString();
  writeFileSync(ACTIVITY, `${JSON.stringify({ read, source: "api.github.com", from: key(from), to: key(today), daily, active, latest }, null, 2)}\n`);
  const stats = {
    read,
    source: "api.github.com",
    orgs: orgs.length,
    repos: own.length,
    stars: starred,
    commits: { year, month: sum(30), week: sum(7) },
    active30: month,
    counted,
    of: live.length,
    contributors,
    private: { commits: priv.commits, people: priv.byLogin },
  };
  writeFileSync(STATS, `${JSON.stringify(stats, null, 2)}\n`);
  console.log(`  oss: ${year} commits in a year, ${stats.commits.week} this week, from ${counted} of ${live.length} repositories pushed in it`);
} catch (err) {
  console.log(`  oss: activity not read (${String(err?.message ?? err)}) — keeping the committed snapshot`);
}