babel/packages/babel-helper-validator-identifier/scripts/generate-identifier-regex.ts

import fs from "node:fs";
// Always use the latest available version of Unicode!
// https://tc39.github.io/ecma262/#sec-conformance
import packageJson from "../package.json" with { type: "json" };
import regenerate from "regenerate";
import assert from "node:assert";

const unicodePackageNamePrefix = "@unicode/unicode-";
// Use findLast to ensure it returns the latest version of Unicode
const unicodePackageName = Object.keys(packageJson.devDependencies).findLast(
  name => name.startsWith(unicodePackageNamePrefix)
);

type UnicodeRange = {
  readonly begin: number;
  readonly end: number;
};

import unicode16Start from "@unicode/unicode-16.0.0/Binary_Property/ID_Start/ranges.js";
import unicode16Cont from "@unicode/unicode-16.0.0/Binary_Property/ID_Continue/ranges.js";

const start = (
  await import(`${unicodePackageName}/Binary_Property/ID_Start/ranges.js`)
).default;
const cont = (
  await import(`${unicodePackageName}/Binary_Property/ID_Continue/ranges.js`)
).default;

function rangesToRegenerate(ranges: readonly UnicodeRange[]): regenerate {
  return ranges.reduce(
    (set: regenerate, { begin, end }: UnicodeRange) =>
      set.addRange(begin, end - 1),
    regenerate()
  );
}

// The minimum Babel 8 required version of Node.js (22.18.0) supports Unicode 16
const unicode16StartSet = rangesToRegenerate(unicode16Start).removeRange(
  0,
  0x7f
);
const unicode16ContSet = rangesToRegenerate(unicode16Cont).removeRange(0, 0x7f);

const unicodeLatestStartSet = rangesToRegenerate(start).removeRange(0, 0x7f);
const unicodeLatestContSet = rangesToRegenerate(cont).removeRange(0, 0x7f);

// Check if the latest Unicode version has removed any characters from ID_Start or ID_Continue that were present in Unicode 16.0.0.
// If so, we may have to add negative lookahead to the regexes in this package. In general this should not happen per Unicode stability policy.
const incompatibleStartSet = unicode16StartSet
  .clone()
  .remove(unicodeLatestStartSet);
const incompatibleContinueSet = unicode16ContSet
  .clone()
  .remove(unicodeLatestContSet);
assert(
  incompatibleStartSet.toArray().length === 0,
  `${unicodePackageName} has removed some characters from ID_Start that were present in Unicode 16.0.0: ${incompatibleStartSet.toString()}`
);
assert(
  incompatibleContinueSet.toArray().length === 0,
  `${unicodePackageName} has removed some characters from ID_Continue that were present in Unicode 16.0.0: ${incompatibleContinueSet.toString()}`
);

const startSet = unicodeLatestStartSet.clone().remove(unicode16StartSet);

const contSetWithoutStart = unicodeLatestContSet
  .clone()
  .remove(unicode16ContSet)
  .remove(startSet);

function esc(code: number) {
  if (code <= 0xff) return "\\x" + code.toString(16).padStart(2, "0");
  return "\\u" + code.toString(16).padStart(4, "0");
}

type IdentifierRegexData = {
  bmp: string;
  supplementary: number[];
};

function generate(regenerateSet: regenerate) {
  const supplementary = [];
  let re = "";
  // @ts-expect-error access internal data property
  const ranges = regenerateSet.data;
  for (let i = 0, at = 0x10000; i < ranges.length; i += 2) {
    const from = ranges[i],
      to = ranges[i + 1] - 1;
    if (to <= 0xffff) {
      if (from === to) re += esc(from);
      else if (from + 1 === to) re += esc(from) + esc(to);
      else re += esc(from) + "-" + esc(to);
    } else {
      supplementary.push(from - at, to - from);
      at = to;
    }
  }
  return { bmp: re, supplementary };
}

function generateIdentifierRegex(
  isBrowser: boolean,
  startData: IdentifierRegexData,
  contData: IdentifierRegexData
) {
  let result = "";
  result += `// Do not edit this file manually. It is generated by \`scripts/generate-identifier-regex.ts\`.
/*
Reason to disable eslint rules:
  - no-misleading-character-class: This rule is disabled because it will check if a combining mark (\\p{M}) is following a base character. It is possible that an identifier char is not following a base character.
  - regexp/no-misleading-unicode-character: The regexp plugin uses Intl.Segmenter to check whether a grapheme cluster can be represented by multiple code points and then report if there is such case. It is possible that two consecutive identifier chars may form a GC, but the intention here is to explicitly form a character set from the given character sets
  - regexp/no-obscure-range: The regexps in this file are soup of "obscure" Unicode characters.
  - regexp/use-ignore-case: This rule is disabled because if we used ignoreCase, the regex will depend on the runtime's Unicode support, which may cause different behavior in different environments.
*/

/* eslint no-misleading-character-class: off, regexp/no-obscure-range: off, regexp/no-misleading-unicode-character: off, regexp/use-ignore-case: off */

// ## Character categories
// Big ugly regular expressions that match characters in the ID_Start / ID_Continue
// binary properties. These are only applied when a character is found
// to actually have a code point larger than 0x80.
// Generated by \`scripts/generate-identifier-regex.ts\`.
`;
  if (isBrowser) {
    result += `/* prettier-ignore */
const bmpIdentifierStartChars = "${startData.bmp}";
/* prettier-ignore */
const bmpIdentifierChars = "${contData.bmp}";
export const bmpIdentifierStart = new RegExp(
  "[" + bmpIdentifierStartChars + "]",
);
export const bmpIdentifier = new RegExp(
  "[" + bmpIdentifierStartChars + bmpIdentifierChars + "]",
);
`;
  } else {
    result += `/* prettier-ignore */
export const bmpIdentifierStart = /[\\p{ID_Start}${startData.bmp}]/u;
/* prettier-ignore */
export const bmpIdentifier = /[\\p{ID_Continue}${startData.bmp}${contData.bmp}]/u;
`;
  }

  result += `
// These are a run-length and offset-encoded representation of the
// >0xffff code points that are a valid part of identifiers. The
// offset starts at 0x10000, and each pair of numbers represents an
// offset to the next range, and then a size of the range. They were
// generated by \`scripts/generate-identifier-regex.ts\`.
/* prettier-ignore */
export const supplementaryIdentifierStartCodes = ${JSON.stringify(startData.supplementary)};
/* prettier-ignore */
export const supplementaryIdentifierCodes = ${JSON.stringify(contData.supplementary)};
`;
  return result;
}

const startData = generate(startSet);
const contData = generate(contSetWithoutStart);
fs.writeFileSync(
  new URL("../src/identifier-regex.ts", import.meta.url),
  generateIdentifierRegex(false, startData, contData)
);

const startDataBrowser = generate(unicodeLatestStartSet);
const contDataBrowser = generate(
  unicodeLatestContSet.clone().remove(unicodeLatestStartSet)
);
fs.writeFileSync(
  new URL("../src/identifier-regex-browser.ts", import.meta.url),
  generateIdentifierRegex(true, startDataBrowser, contDataBrowser)
);