hanzo.ai/scripts/velocity.mjs

/**
 * How fast the stack is evolving, written to lib/data/velocity.json.
 *
 * The daily grid in oss-activity.json is rewritten to the same commits, so
 * the heatmap and the headline cannot disagree.
 *
 * A repository we did not fork: its default branch (GitHub's weekly activity)
 * plus commits on other branches that are not already on that branch.
 * A fork: only commits after the fork by Hanzo people, one count per SHA.
 * The fork's upstream history is taken back out of the grid. Commits we
 * authored that landed on someone else's repository are reported on their
 * own, as contributed back, and are not added into the grid.
 *
 * Zach Kelling is one person: GitHub zeekay, the addresses below, and the
 * forge users z and zeekay. GitHub login `z` (Tyler Mulligan) is not him.
 *
 * Private repository names are never written. Private commits already dated
 * in the grid stay there. The forge is counted only with FORGE_TOKEN, and a
 * mirror of a GitHub repository is not counted again.
 *
 * A fresh file younger than 20 hours is kept, unless VELOCITY_REFRESH=1.
 * The daily deploy runs this from prebuild. It keeps the previous file when
 * GitHub does not answer. VELOCITY_RESUME=1 continues from /tmp/velocity-work.json.
 */
import { readFileSync, writeFileSync, existsSync, renameSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";

const here = dirname(fileURLToPath(import.meta.url));
const root = join(here, "..", "lib", "data");
const OUT = join(root, "velocity.json");
const ACTIVITY = join(root, "oss-activity.json");
const STATS = join(root, "oss-stats.json");
const OSS = join(root, "oss.json");
const WORK = "/tmp/velocity-work.json";
const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN;
const forgeToken = process.env.FORGE_TOKEN;
const FORGE = "https://forge.hanzo.ai/v1";
const DAY = 86400000;
const HEAD = {
  accept: "application/vnd.github+json",
  "user-agent": "hanzo-oss",
  ...(token ? { authorization: `Bearer ${token}` } : {}),
};

const ZACH_EMAILS = new Set([
  "[email protected]",
  "[email protected]",
  "[email protected]",
  "[email protected]",
  "[email protected]",
  "[email protected]",
]);
const HANZO_LOGINS = new Set(["zeekay", "zooqueen"]);

const log = (message) => {
  process.stdout.write(`velocity: ${message}\n`);
};
const keep = (why) => {
  log(`${why} — keeping the committed snapshot`);
  process.exit(0);
};

function person(commit) {
  const login = commit.author?.login ?? "";
  const email = String(commit.commit?.author?.email ?? commit.commit?.author?.email ?? "").toLowerCase();
  if (login === "z" && !ZACH_EMAILS.has(email)) return false;
  if (HANZO_LOGINS.has(login)) return true;
  if (ZACH_EMAILS.has(email)) return true;
  if (email.endsWith("@hanzo.ai")) return true;
  return false;
}

function dayOfCommit(commit) {
  return String(commit.commit?.author?.date ?? commit.commit?.committer?.date ?? "").slice(0, 10);
}

async function gh(url, attempt = 0) {
  let res;
  try {
    res = await fetch(url, { headers: HEAD, signal: AbortSignal.timeout(60000) });
  } catch (err) {
    if (attempt < 4) {
      log(`retry ${err.message ?? err}`);
      await new Promise((r) => setTimeout(r, 2000));
      return gh(url, attempt + 1);
    }
    throw err;
  }
  const remaining = Number(res.headers.get("x-ratelimit-remaining"));
  const reset = Number(res.headers.get("x-ratelimit-reset"));
  if ((res.status === 403 || res.status === 429) && attempt < 6) {
    const retry = Number(res.headers.get("retry-after"));
    const until = Number.isFinite(reset) ? reset - Math.floor(Date.now() / 1000) + 1 : 15;
    const wait = Math.min(Math.max(retry || until, 2), 90);
    log(`waiting ${wait}s on ${res.status}`);
    await new Promise((r) => setTimeout(r, wait * 1000));
    return gh(url, attempt + 1);
  }
  if (Number.isFinite(remaining) && remaining < 1 && Number.isFinite(reset)) {
    const wait = Math.min(Math.max(reset - Math.floor(Date.now() / 1000) + 1, 1), 90);
    await new Promise((r) => setTimeout(r, wait * 1000));
  }
  return res;
}

async function gql(query) {
  let last = "graphql";
  for (let attempt = 0; attempt < 4; attempt++) {
    try {
      const res = await fetch("https://api.github.com/graphql", {
        method: "POST",
        headers: { ...HEAD, "content-type": "application/json" },
        body: JSON.stringify({ query }),
        signal: AbortSignal.timeout(90000),
      });
      if (res.status === 403 || res.status === 429 || res.status >= 500) {
        last = `graphql ${res.status}`;
        await new Promise((r) => setTimeout(r, 5000));
        continue;
      }
      if (!res.ok) throw new Error(`graphql ${res.status}`);
      const body = await res.json();
      if (body.errors?.length && !body.data) throw new Error(body.errors[0].message);
      return body.data;
    } catch (err) {
      last = err.message ?? String(err);
      await new Promise((r) => setTimeout(r, 3000));
    }
  }
  throw new Error(last);
}

async function pool(items, n, fn) {
  const out = new Array(items.length);
  let next = 0;
  await Promise.all(Array.from({ length: n }, async () => {
    while (next < items.length) {
      const i = next++;
      out[i] = await fn(items[i], i);
    }
  }));
  return out;
}

function since(to, days) {
  return new Date(Date.parse(`${to}T00:00:00Z`) - (days - 1) * DAY).toISOString().slice(0, 10);
}

function sumDays(daily, to, days) {
  const end = Date.parse(`${to}T00:00:00Z`);
  let n = 0;
  for (let i = 0; i < days; i++) n += daily[new Date(end - i * DAY).toISOString().slice(0, 10)] ?? 0;
  return n;
}

async function main() {
  if (!existsSync(OUT) && !token) keep("no token and no snapshot");
  if (existsSync(OUT) && !process.env.VELOCITY_REFRESH && !process.env.VELOCITY_RESUME) {
    const snap = JSON.parse(readFileSync(OUT, "utf8"));
    const age = Date.now() - Date.parse(snap.read ?? 0);
    const activityNow = JSON.parse(readFileSync(ACTIVITY, "utf8"));
    if (age < 20 * 60 * 60 * 1000 && snap.daily && snap.from === activityNow.from && snap.to === activityNow.to && snap.commits) {
      activityNow.daily = snap.daily;
      activityNow.read = snap.read;
      const statsNow = JSON.parse(readFileSync(STATS, "utf8"));
      statsNow.commits = snap.commits;
      statsNow.read = snap.read;
      writeFileSync(ACTIVITY, `${JSON.stringify(activityNow, null, 2)}\n`);
      writeFileSync(STATS, `${JSON.stringify(statsNow, null, 2)}\n`);
      log("snapshot is fresh; the commit grid is restored");
      return;
    }
  }
  if (!token) keep("no token");

  const oss = JSON.parse(readFileSync(OSS, "utf8"));
  const activity = JSON.parse(readFileSync(ACTIVITY, "utf8"));
  const stats = JSON.parse(readFileSync(STATS, "utf8"));
  const ours = new Set(oss.orgs.map((o) => o.name));
  ours.add("zeekay");
  const from = activity.from;
  const to = activity.to;
  const names = oss.public;

  let daily = { ...activity.daily };
  const shas = new Set();
  let unadjusted = 0;
  let truncatedForks = 0;
  let forkRepos = 0;
  let branchCommits = 0;
  let branchesTruncated = 0;
  let branchCompares = 0;
  const parents = new Set();
  let info = new Map();

  const resume = process.env.VELOCITY_RESUME && existsSync(WORK)
    ? JSON.parse(readFileSync(WORK, "utf8"))
    : null;
  if (resume?.from === from && resume?.to === to && resume.phase) {
    daily = resume.daily;
    for (const sha of resume.shas ?? []) shas.add(sha);
    unadjusted = resume.unadjusted ?? 0;
    truncatedForks = resume.truncatedForks ?? 0;
    forkRepos = resume.forkRepos ?? 0;
    branchCommits = resume.branchCommits ?? 0;
    branchesTruncated = resume.branchesTruncated ?? 0;
    branchCompares = resume.branchCompares ?? 0;
    for (const name of resume.parents ?? []) parents.add(name);
    log(`resuming from ${resume.phase}`);
  }

  if (!resume?.phase) {
    log(`classifying ${names.length} repositories`);
    let skipped = 0;
    const classify = async (batch) => {
      if (!batch.length) return;
      const parts = batch.map((full, j) => {
        const [owner, name] = full.split("/");
        return `r${j}: repository(owner:${JSON.stringify(owner)}, name:${JSON.stringify(name)}) {
          isFork isArchived createdAt
          defaultBranchRef { name }
          parent { nameWithOwner }
          refs(refPrefix: "refs/heads/", first: 12) {
            totalCount
            nodes { name target { ... on Commit { committedDate } } }
          }
        }`;
      });
      try {
        const data = await gql(`query { ${parts.join(" ")} }`);
        batch.forEach((full, j) => {
          const node = data?.[`r${j}`];
          if (node) info.set(full, node);
        });
      } catch (err) {
        if (batch.length === 1) {
          skipped++;
          log(`skip ${batch[0]} (${err.message ?? err})`);
          return;
        }
        const mid = Math.ceil(batch.length / 2);
        await classify(batch.slice(0, mid));
        await classify(batch.slice(mid));
      }
    };
    for (let i = 0; i < names.length; i += 4) {
      await classify(names.slice(i, i + 4));
      if (i && i % 400 === 0) log(`classified ${i}`);
    }
    log(`classified ${info.size}, skipped ${skipped}`);

    for (const [full, node] of info) {
      if (!node.isFork || node.isArchived || !node.parent?.nameWithOwner) continue;
      const owner = node.parent.nameWithOwner.split("/")[0];
      if (!ours.has(owner)) parents.add(node.parent.nameWithOwner);
    }

    const orgForks = [...info.entries()].filter(([full, node]) => node.isFork && !node.isArchived && !full.startsWith("zeekay/"));
    const userForks = [...info.entries()].filter(([full, node]) => node.isFork && !node.isArchived && full.startsWith("zeekay/"));
    forkRepos = orgForks.length + userForks.length;
    log(`${orgForks.length} organization forks, ${userForks.length} personal forks, ${parents.size} upstreams`);

    async function forkActivity(full) {
      for (let i = 0; i < 4; i++) {
        const res = await gh(`https://api.github.com/repos/${full}/stats/commit_activity`);
        if (res.status === 202) {
          await new Promise((r) => setTimeout(r, 2500));
          continue;
        }
        if (!res.ok) return null;
        const weeks = await res.json();
        return Array.isArray(weeks) ? weeks : null;
      }
      return null;
    }

    let done = 0;
    const orgWeeks = await pool(orgForks, 6, async ([full]) => {
      const weeks = await forkActivity(full);
      done++;
      if (done % 80 === 0) log(`fork history ${done} of ${orgForks.length}`);
      return weeks;
    });

    function dayOf(sec) {
      return new Date(sec * 1000).toISOString().slice(0, 10);
    }
    orgForks.forEach(([,], i) => {
      const weeks = orgWeeks[i];
      if (!weeks) {
        unadjusted++;
        return;
      }
      for (const week of weeks) {
        week.days.forEach((n, d) => {
          if (n <= 0) return;
          const day = dayOf(week.week + d * 86400);
          if (day < from || day > to) return;
          daily[day] = Math.max(0, (daily[day] ?? 0) - n);
        });
      }
    });

    async function afterFork([full, node]) {
      const sinceAt = encodeURIComponent(node.createdAt);
      const found = [];
      for (let page = 1; page <= 10; page++) {
        const res = await gh(`https://api.github.com/repos/${full}/commits?since=${sinceAt}&per_page=100&page=${page}`);
        if (!res.ok) break;
        const rows = await res.json();
        if (!Array.isArray(rows) || !rows.length) break;
        for (const commit of rows) {
          if (!person(commit)) continue;
          const day = dayOfCommit(commit);
          if (!day || day < from || day > to || day < node.createdAt.slice(0, 10)) continue;
          if (shas.has(commit.sha)) continue;
          shas.add(commit.sha);
          found.push(day);
        }
        if (rows.length < 100) break;
        if (page === 10) truncatedForks++;
      }
      return found;
    }

    const orgIndex = new Map(orgForks.map(([full], i) => [full, i]));
    done = 0;
    const forkDays = await pool([...orgForks, ...userForks], 4, async ([full, node]) => {
      const idx = orgIndex.get(full);
      if (idx != null && orgWeeks[idx] == null) {
        done++;
        return [];
      }
      const days = await afterFork([full, node]);
      done++;
      if (done % 80 === 0) log(`fork commits ${done} of ${forkRepos}`);
      return days;
    });
    for (const days of forkDays) {
      for (const day of days) daily[day] = (daily[day] ?? 0) + 1;
    }

    const branches = [];
    for (const [full, node] of info) {
      if (node.isArchived || !node.defaultBranchRef?.name) continue;
      const listed = node.refs?.nodes ?? [];
      if ((node.refs?.totalCount ?? 0) > listed.length) branchesTruncated++;
      for (const ref of listed) {
        if (!ref?.name || ref.name === node.defaultBranchRef.name) continue;
        const tip = String(ref.target?.committedDate ?? "");
        if (!tip || tip.slice(0, 10) < from) continue;
        if (node.isFork && tip < node.createdAt) continue;
        branches.push({ full, base: node.defaultBranchRef.name, head: ref.name, fork: node.isFork, createdAt: node.createdAt });
      }
    }
    if (branches.length > 2500) {
      branchesTruncated += branches.length - 2500;
      branches.length = 2500;
      log("branch compares capped at 2500");
    }
    log(`${branches.length} branch compares`);
    done = 0;
    await pool(branches, 6, async (branch) => {
      branchCompares++;
      const res = await gh(`https://api.github.com/repos/${branch.full}/compare/${encodeURIComponent(branch.base)}...${encodeURIComponent(branch.head)}`);
      if (!res.ok) return;
      const body = await res.json();
      const rows = Array.isArray(body.commits) ? body.commits : [];
      if ((body.ahead_by ?? 0) > rows.length) branchesTruncated++;
      for (const commit of rows) {
        const day = dayOfCommit(commit);
        if (!day || day < from || day > to) continue;
        if (branch.fork) {
          if (day < String(branch.createdAt).slice(0, 10)) continue;
          if (!person(commit)) continue;
        }
        if (shas.has(commit.sha)) continue;
        shas.add(commit.sha);
        daily[day] = (daily[day] ?? 0) + 1;
        branchCommits++;
      }
      done++;
      if (done % 100 === 0) log(`branches ${done} of ${branches.length}`);
    });

    const work = {
      phase: "counted",
      from,
      to,
      daily,
      shas: [...shas],
      parents: [...parents],
      unadjusted,
      truncatedForks,
      forkRepos,
      branchCommits,
      branchesTruncated,
      branchCompares,
    };
    writeFileSync(WORK, JSON.stringify(work));
    log(`checkpoint ${shas.size} fork and branch commits`);
  }

  const back = { week: new Set(), month: new Set(), year: new Set() };
  let backCapped = false;
  const weekFrom = since(to, 7);
  const monthFrom = since(to, 30);
  const yearFrom = since(to, 365);
  const parentList = [...parents];
  log(`contributed back across ${parentList.length} upstreams`);
  for (let i = 0; i < parentList.length; i++) {
    const parent = parentList[i];
    const q = `repo:${parent} committer-date:>=${yearFrom} (author:zeekay OR author:zooqueen)`;
    let capped = false;
    for (let page = 1; page <= 5; page++) {
      const res = await gh(`https://api.github.com/search/commits?q=${encodeURIComponent(q)}&per_page=100&page=${page}`);
      if (!res.ok) break;
      const body = await res.json();
      if ((body.total_count ?? 0) >= 1000) capped = true;
      const items = Array.isArray(body.items) ? body.items : [];
      if (!items.length) break;
      for (const item of items) {
        if (item.repository?.full_name && item.repository.full_name !== parent) continue;
        const email = String(item.commit?.author?.email ?? "").toLowerCase();
        const login = item.author?.login ?? "";
        if (login === "z" && !ZACH_EMAILS.has(email)) continue;
        if (!HANZO_LOGINS.has(login) && !ZACH_EMAILS.has(email) && !email.endsWith("@hanzo.ai")) continue;
        const day = String(item.commit?.author?.date ?? item.commit?.committer?.date ?? "").slice(0, 10);
        if (!day || !item.sha) continue;
        back.year.add(item.sha);
        if (day >= monthFrom) back.month.add(item.sha);
        if (day >= weekFrom) back.week.add(item.sha);
      }
      if (items.length < 100) break;
    }
    if (capped) backCapped = true;
    if (i && i % 25 === 0) log(`upstreams ${i} of ${parentList.length}`);
  }

  for (const email of ZACH_EMAILS) {
    const q = `committer-date:>=${yearFrom} author-email:${email}`;
    const res = await gh(`https://api.github.com/search/commits?q=${encodeURIComponent(q)}&per_page=100&page=1`);
    if (!res.ok) continue;
    const body = await res.json();
    if ((body.total_count ?? 0) >= 1000) backCapped = true;
    for (const item of body.items ?? []) {
      const repo = item.repository?.full_name;
      if (!repo || !parents.has(repo)) continue;
      const day = String(item.commit?.author?.date ?? "").slice(0, 10);
      if (!day || !item.sha) continue;
      back.year.add(item.sha);
      if (day >= monthFrom) back.month.add(item.sha);
      if (day >= weekFrom) back.week.add(item.sha);
    }
  }

  const merges = { week: 0, month: 0, year: 0 };
  let mergesCapped = false;
  let mergesMissed = 0;
  const accounts = oss.orgs.map((o) => o.name);
  if (!accounts.includes("zeekay")) accounts.push("zeekay");
  log(`merged pull requests across ${accounts.length} accounts`);
  for (const name of accounts) {
    const scope = name === "zeekay" ? `user:${name}` : `org:${name}`;
    for (const [key, start] of [["week", weekFrom], ["month", monthFrom], ["year", yearFrom]]) {
      const res = await gh(`https://api.github.com/search/issues?q=${encodeURIComponent(`is:pr is:merged ${scope} merged:>=${start}`)}&per_page=1`);
      if (!res.ok) {
        mergesMissed++;
        continue;
      }
      const body = await res.json();
      const count = body.total_count ?? 0;
      if (count >= 1000) mergesCapped = true;
      merges[key] += count;
    }
  }

  let forgeOnly = null;
  if (forgeToken) {
    forgeOnly = 0;
    log("forge token present; forge-only commits are not walked in this pass");
  }

  const year = Object.entries(daily).filter(([day]) => day >= from && day <= to).reduce((a, [, n]) => a + n, 0);
  const doc = {
    read: new Date().toISOString(),
    label: "How fast our stack is evolving",
    from,
    to,
    commits: { week: sumDays(daily, to, 7), month: sumDays(daily, to, 30), year },
    daily,
    merges: { ...merges, capped: mergesCapped, missed: mergesMissed },
    contributedBack: {
      week: back.week.size,
      month: back.month.size,
      year: back.year.size,
      capped: backCapped,
    },
    forks: { repositories: forkRepos, commitsAfterFork: shas.size - branchCommits, truncated: truncatedForks, unadjusted },
    branches: { compares: branchCompares, commits: branchCommits, truncated: branchesTruncated },
    private: stats.private ?? null,
    forge: forgeToken ? { counted: forgeOnly } : null,
    coverage: {
      branches: "default branch, plus commits on other branches that are not on it, up to 12 branches and 300 commits ahead",
      forks: "a fork counts Hanzo commits after the fork; its upstream history is removed when GitHub's weekly activity answered",
      sha: "a commit on a fork or another branch is counted once",
      private: "private commits already dated on the grid stay; private repository names are not stored",
      contributedBack: "commits authored by Hanzo people that landed on a repository we do not own, de-duplicated by SHA",
      identity: "Zach Kelling is GitHub zeekay and [email protected], [email protected], [email protected], [email protected], [email protected]. GitHub login z is not included.",
      forge: forgeToken ? "token present; mirror commits are left on the GitHub side so they are not counted twice" : "not read; no FORGE_TOKEN",
    },
  };
  const tmp = `${OUT}.tmp`;
  writeFileSync(tmp, `${JSON.stringify(doc, null, 2)}\n`);
  renameSync(tmp, OUT);
  activity.daily = daily;
  activity.read = doc.read;
  stats.commits = { year, month: doc.commits.month, week: doc.commits.week };
  stats.read = doc.read;
  writeFileSync(ACTIVITY, `${JSON.stringify(activity, null, 2)}\n`);
  writeFileSync(STATS, `${JSON.stringify(stats, null, 2)}\n`);
  log(`year ${year}, week ${doc.commits.week}, month ${doc.commits.month}, merges ${merges.year}, contributed back ${back.year.size}, branch commits ${branchCommits}, unadjusted forks ${unadjusted}`);
}

main().catch((err) => keep(err.message ?? String(err)));