hanzo.ai/scripts/velocity-slices.mjs

/**
 * Commits and merged pull requests for the past year, on each public
 * repository, each organization, the whole set, and zeekay.
 *
 * zeekay's work is inside the repository and organization totals. His own
 * line is the same commits and merges, counted again so the person is visible.
 * A fork counts Hanzo commits after the fork. Private repository names are
 * not written. The daily deploy runs this after velocity. A file younger
 * than 20 hours is kept unless VELOCITY_REFRESH=1.
 */
import { readFileSync, writeFileSync, existsSync, renameSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";

const here = dirname(fileURLToPath(import.meta.url));
const root = join(here, "..", "lib", "data");
const VELOCITY = join(root, "velocity.json");
const REPOS = join(root, "velocity-repos.json");
const OSS = join(root, "oss.json");
const WORK = "/tmp/velocity-slices-work.json";
const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN;
const DAY = 86400000;
const HEAD = {
  accept: "application/vnd.github+json",
  "user-agent": "hanzo-oss",
  ...(token ? { authorization: `Bearer ${token}` } : {}),
};
const ZACH_EMAILS = new Set([
  "[email protected]",
  "[email protected]",
  "[email protected]",
  "[email protected]",
  "[email protected]",
  "[email protected]",
]);
const HANZO_LOGINS = new Set(["zeekay", "zooqueen"]);

const log = (message) => process.stdout.write(`slices: ${message}\n`);
const blank = () => ({ week: 0, month: 0, year: 0 });

function person(commit) {
  const login = commit.author?.login ?? "";
  const email = String(commit.commit?.author?.email ?? "").toLowerCase();
  if (login === "z" && !ZACH_EMAILS.has(email)) return false;
  if (HANZO_LOGINS.has(login)) return true;
  if (ZACH_EMAILS.has(email)) return true;
  if (email.endsWith("@hanzo.ai")) return true;
  return false;
}

async function jsonBody(res) {
  const text = await res.text();
  if (!text) return null;
  try { return JSON.parse(text); }
  catch { return null; }
}

async function gh(url, attempt = 0) {
  let res;
  try {
    res = await fetch(url, { headers: HEAD, signal: AbortSignal.timeout(60000) });
  } catch (err) {
    if (attempt < 4) {
      await new Promise((r) => setTimeout(r, 2000));
      return gh(url, attempt + 1);
    }
    throw err;
  }
  const remaining = Number(res.headers.get("x-ratelimit-remaining"));
  const reset = Number(res.headers.get("x-ratelimit-reset"));
  if ((res.status === 403 || res.status === 429) && attempt < 6) {
    const wait = Math.min(Math.max(Number(res.headers.get("retry-after")) || (Number.isFinite(reset) ? reset - Math.floor(Date.now() / 1000) + 1 : 15), 2), 90);
    log(`waiting ${wait}s`);
    await new Promise((r) => setTimeout(r, wait * 1000));
    return gh(url, attempt + 1);
  }
  if (Number.isFinite(remaining) && remaining < 1 && Number.isFinite(reset)) {
    const wait = Math.min(Math.max(reset - Math.floor(Date.now() / 1000) + 1, 1), 90);
    await new Promise((r) => setTimeout(r, wait * 1000));
  }
  return res;
}

async function gql(query) {
  const res = await fetch("https://api.github.com/graphql", {
    method: "POST",
    headers: { ...HEAD, "content-type": "application/json" },
    body: JSON.stringify({ query }),
    signal: AbortSignal.timeout(90000),
  });
  if (!res.ok) throw new Error(`graphql ${res.status}`);
  const body = await jsonBody(res);
  if (!body?.data) throw new Error(body?.errors?.[0]?.message ?? "graphql empty");
  return body.data;
}

async function pool(items, n, fn) {
  const out = new Array(items.length);
  let next = 0;
  await Promise.all(Array.from({ length: n }, async () => {
    while (next < items.length) {
      const i = next++;
      try { out[i] = await fn(items[i], i); }
      catch (err) { log(`skip ${err.message ?? err}`); }
    }
  }));
  return out;
}

function bump(box, day, weekFrom, monthFrom, from, to) {
  if (!day || day < from || day > to) return;
  box.year += 1;
  if (day >= monthFrom) box.month += 1;
  if (day >= weekFrom) box.week += 1;
}

async function main() {
  if (!token) {
    log("no token — keeping the committed snapshot");
    return;
  }
  const velocity = existsSync(VELOCITY) ? JSON.parse(readFileSync(VELOCITY, "utf8")) : null;
  if (!velocity?.from || !velocity?.to) {
    log("no velocity window — run velocity first");
    return;
  }
  const age = Date.now() - Date.parse(velocity.slicesRead ?? 0);
  if (!process.env.VELOCITY_REFRESH && !process.env.VELOCITY_RESUME && velocity.slicesRead && age < 20 * 60 * 60 * 1000) {
    log("snapshot is fresh");
    return;
  }
  const from = velocity.from;
  const to = velocity.to;
  const weekFrom = new Date(Date.parse(`${to}T00:00:00Z`) - 6 * DAY).toISOString().slice(0, 10);
  const monthFrom = new Date(Date.parse(`${to}T00:00:00Z`) - 29 * DAY).toISOString().slice(0, 10);
  const oss = JSON.parse(readFileSync(OSS, "utf8"));
  const names = oss.public;
  const accounts = oss.orgs.map((o) => o.name);

  let info = new Map();
  const counts = new Map();
  const resume = process.env.VELOCITY_RESUME && existsSync(WORK) ? JSON.parse(readFileSync(WORK, "utf8")) : null;
  if (resume?.from === from && resume?.to === to && resume.phase === "commits") {
    for (const [full, row] of Object.entries(resume.counts)) counts.set(full, row);
    log(`resuming commits for ${counts.size} repositories`);
  } else {
    log(`classifying ${names.length} repositories`);
    for (let i = 0; i < names.length; i += 10) {
      const batch = names.slice(i, i + 10);
      const parts = batch.map((full, j) => {
        const [owner, name] = full.split("/");
        return `r${j}: repository(owner:${JSON.stringify(owner)}, name:${JSON.stringify(name)}) { isFork isArchived createdAt }`;
      });
      try {
        const data = await gql(`query { ${parts.join(" ")} }`);
        batch.forEach((full, j) => {
          if (data?.[`r${j}`]) info.set(full, data[`r${j}`]);
        });
      } catch (err) {
        log(`classify skip ${batch[0]} (${err.message ?? err})`);
      }
      if (i && i % 500 === 0) log(`classified ${i}`);
    }

    const live = names.filter((full) => !info.get(full)?.isArchived);
    const forks = live.filter((full) => info.get(full)?.isFork);
    const rest = live.filter((full) => !info.get(full)?.isFork);
    log(`${rest.length} repositories, ${forks.length} forks`);

    async function activity(full) {
      for (let i = 0; i < 3; i++) {
        const res = await gh(`https://api.github.com/repos/${full}/stats/commit_activity`);
        if (res.status === 202) {
          await new Promise((r) => setTimeout(r, 2000));
          continue;
        }
        if (!res.ok) return null;
        const weeks = await jsonBody(res);
        return Array.isArray(weeks) ? weeks : null;
      }
      return null;
    }

    let done = 0;
    await pool(rest, 8, async (full) => {
      const weeks = await activity(full);
      const box = { commits: blank(), merges: blank(), zeekayMerges: blank(), fork: false };
      if (weeks) {
        for (const week of weeks) {
          week.days.forEach((n, d) => {
            if (n <= 0) return;
            const day = new Date((week.week + d * 86400) * 1000).toISOString().slice(0, 10);
            if (day < from || day > to) return;
            box.commits.year += n;
            if (day >= monthFrom) box.commits.month += n;
            if (day >= weekFrom) box.commits.week += n;
          });
        }
      }
      counts.set(full, box);
      done++;
      if (done % 400 === 0) log(`activity ${done} of ${rest.length}`);
    });

    done = 0;
    await pool(forks, 4, async (full) => {
      const node = info.get(full);
      const box = { commits: blank(), merges: blank(), zeekayMerges: blank(), fork: true };
      const sinceAt = encodeURIComponent(node?.createdAt ?? `${from}T00:00:00Z`);
      const seen = new Set();
      for (let page = 1; page <= 10; page++) {
        const res = await gh(`https://api.github.com/repos/${full}/commits?since=${sinceAt}&per_page=100&page=${page}`);
        if (!res.ok) break;
        const rows = await jsonBody(res);
        if (!Array.isArray(rows) || !rows.length) break;
        for (const commit of rows) {
          if (!person(commit) || seen.has(commit.sha)) continue;
          const day = String(commit.commit?.author?.date ?? commit.commit?.committer?.date ?? "").slice(0, 10);
          if (!day || day < from || day > to || day < String(node?.createdAt ?? "").slice(0, 10)) continue;
          seen.add(commit.sha);
          bump(box.commits, day, weekFrom, monthFrom, from, to);
        }
        if (rows.length < 100) break;
        if (page === 10) box.truncated = true;
      }
      counts.set(full, box);
      done++;
      if (done % 80 === 0) log(`fork commits ${done} of ${forks.length}`);
    });

    writeFileSync(WORK, JSON.stringify({ phase: "commits", from, to, counts: Object.fromEntries(counts) }));
    log(`checkpoint ${counts.size} repositories`);
  }

  const zeekay = { commits: { year: 0 }, merges: blank(), capped: false, privateCommits: velocity.private?.people?.zeekay ?? 0 };
  const listed = new Set(names);
  const unlisted = new Map();
  const byZeekay = new Map();
  const addMerge = (box, day) => {
    if (!day || day < from || day > to) {
      box.year += 1;
      return;
    }
    bump(box, day, weekFrom, monthFrom, from, to);
  };
  log(`merged pull requests across ${accounts.length} accounts`);
  for (const name of accounts) {
    const scope = name === "zeekay" ? `user:${name}` : `org:${name}`;
    const q = `is:pr is:merged ${scope} merged:>=${from}`;
    for (let page = 1; page <= 10; page++) {
      const res = await gh(`https://api.github.com/search/issues?q=${encodeURIComponent(q)}&per_page=100&page=${page}`);
      if (!res.ok) {
        log(`merges miss ${name} ${res.status}`);
        break;
      }
      const body = await jsonBody(res);
      if (!body) break;
      const items = Array.isArray(body.items) ? body.items : [];
      for (const item of items) {
        const full = String(item.repository_url ?? "").split("/repos/")[1];
        const day = String(item.closed_at ?? item.pull_request?.merged_at ?? "").slice(0, 10);
        const login = item.user?.login ?? "";
        const named = full && listed.has(full);
        if (!named) {
          const box = unlisted.get(name) ?? blank();
          addMerge(box, day);
          unlisted.set(name, box);
        } else {
          if (!counts.has(full)) counts.set(full, { commits: blank(), merges: blank(), zeekayMerges: blank(), fork: false });
          const row = counts.get(full);
          addMerge(row.merges, day);
          if (login === "zeekay") addMerge(row.zeekayMerges, day);
        }
        if (login === "zeekay") addMerge(zeekay.merges, day);
      }
      if (items.length < 100) break;
    }
  }

  log("zeekay commits by account");
  for (const name of accounts) {
    const scope = name === "zeekay" ? `user:${name}` : `org:${name}`;
    const q = `${scope} author:zeekay committer-date:>=${from}`;
    const res = await gh(`https://api.github.com/search/commits?q=${encodeURIComponent(q)}&per_page=1`);
    if (!res.ok) {
      log(`zeekay commits miss ${name} ${res.status}`);
      zeekay.capped = true;
      byZeekay.set(name, { year: 0, capped: true });
      continue;
    }
    const body = await jsonBody(res);
    const total = body?.total_count ?? 0;
    const capped = body?.incomplete_results === true;
    if (capped) zeekay.capped = true;
    zeekay.commits.year += total;
    byZeekay.set(name, { year: total, capped });
  }

  const orgs = [];
  for (const name of accounts) {
    const rows = [...counts.entries()].filter(([full]) => full.startsWith(`${name}/`));
    const commits = blank();
    const merges = blank();
    let repositories = 0;
    for (const [, row] of rows) {
      if (row.commits.year || row.merges.year) repositories++;
      for (const key of ["week", "month", "year"]) {
        commits[key] += row.commits[key];
        merges[key] += row.merges[key];
      }
    }
    const hidden = unlisted.get(name);
    if (hidden) {
      for (const key of ["week", "month", "year"]) merges[key] += hidden[key];
    }
    const his = byZeekay.get(name);
    if (!commits.year && !merges.year && !his?.year) continue;
    orgs.push({
      name,
      commits,
      merges,
      repositories,
      ...(hidden?.year ? { unlistedMerges: hidden.year } : {}),
      ...(his ? { zeekayCommits: his } : {}),
    });
  }
  orgs.sort((a, b) => b.commits.year - a.commits.year || b.merges.year - a.merges.year);

  const kept = {};
  for (const [full, row] of counts) {
    if (!row.commits.year && !row.merges.year) continue;
    kept[full] = {
      commits: row.commits,
      merges: row.merges,
      ...(row.zeekayMerges.year ? { zeekayMerges: row.zeekayMerges } : {}),
      ...(row.fork ? { fork: true } : {}),
      ...(row.truncated ? { truncated: true } : {}),
    };
  }

  const read = new Date().toISOString();
  const repoDoc = {
    read,
    from,
    to,
    label: "Commits and merged pull requests in the past year, each public repository",
    coverage: "Default branch. A fork counts Hanzo commits after the fork. zeekay is included in the repository. Private repository names are not stored. Commits that are only on another branch stay in the global headline.",
    repositories: Object.keys(kept).length,
    repos: kept,
  };
  const tmp = `${REPOS}.tmp`;
  writeFileSync(tmp, `${JSON.stringify(repoDoc, null, 2)}\n`);
  renameSync(tmp, REPOS);

  velocity.orgs = orgs;
  velocity.zeekay = zeekay;
  velocity.slicesRead = read;
  velocity.slices = {
    repositories: Object.keys(kept).length,
    commits: orgs.reduce((n, o) => n + o.commits.year, 0),
    merges: orgs.reduce((n, o) => n + o.merges.year, 0),
  };
  writeFileSync(VELOCITY, `${JSON.stringify(velocity, null, 2)}\n`);
  log(`orgs ${orgs.length}, repositories ${Object.keys(kept).length}, merges ${velocity.slices.merges}, zeekay commits ${zeekay.commits.year}, zeekay merges ${zeekay.merges.year}`);
}

main().catch((err) => {
  log(`${err.message ?? err} — keeping the committed snapshot`);
});