datastore/ci/jobs/scripts/docs/check_autogenerated_regions.py

#!/usr/bin/env python3
"""Reject direct edits to content fenced by AUTOGENERATED markers.

The docs generators preserve hand-written content outside these regions and
replace everything inside them. Pull requests should therefore change the
generator's source data rather than editing generated output directly.

The check compares marked regions at two Git revisions. Rename detection keeps
an unchanged generated region valid when its containing file is moved.
Localized docs trees are excluded: they are static translations of the English
source docs rather than independently generated source content.
"""

import argparse
import re
import subprocess
import sys

START_PATTERN = (
    rb"(?:<!--\s*AUTOGENERATED_START\s*-->|\{/\*\s*AUTOGENERATED_START\s*\*/\})"
)
END_PATTERN = rb"(?:<!--\s*AUTOGENERATED_END\s*-->|\{/\*\s*AUTOGENERATED_END\s*\*/\})"
BLOCK_RE = re.compile(START_PATTERN + rb"(.*?)" + END_PATTERN, re.DOTALL)
LOCALIZED_DOCS_PREFIXES = tuple(
    f"docs/{locale}/" for locale in ("ar", "es", "fr", "ja", "ko", "pt-BR", "ru", "zh")
)


def _git(*args):
    result = subprocess.run(["git", *args], capture_output=True, check=False)
    if result.returncode:
        error = result.stderr.decode("utf-8", errors="replace").strip()
        raise RuntimeError(f"git {' '.join(args)} failed: {error}")
    return result.stdout


def _resolve_ref(ref):
    return (
        _git("rev-parse", "--verify", "--end-of-options", f"{ref}^{{commit}}")
        .decode()
        .strip()
    )


def _changed_docs(base_ref, head_ref):
    """Return changed, non-localized docs as (old_path, new_path) pairs."""
    output = _git(
        "diff",
        "--name-status",
        "-z",
        "--find-renames",
        base_ref,
        head_ref,
        "--",
        "docs/",
    )
    fields = output.rstrip(b"\0").split(b"\0") if output else []
    changed = []
    index = 0
    while index < len(fields):
        status = fields[index].decode("ascii")
        index += 1
        kind = status[0]
        if kind in ("R", "C"):
            old_path = fields[index].decode("utf-8", errors="surrogateescape")
            new_path = fields[index + 1].decode("utf-8", errors="surrogateescape")
            index += 2
        else:
            path = fields[index].decode("utf-8", errors="surrogateescape")
            index += 1
            old_path = None if kind == "A" else path
            new_path = None if kind == "D" else path

        if old_path and old_path.startswith(LOCALIZED_DOCS_PREFIXES):
            old_path = None
        if new_path and new_path.startswith(LOCALIZED_DOCS_PREFIXES):
            new_path = None
        if old_path is None and new_path is None:
            continue
        changed.append((old_path, new_path))
    return changed


def _read_file(ref, path):
    if path is None:
        return b""
    return _git("show", f"{ref}:{path}")


def _generated_blocks(content):
    # Only complete marker pairs define a protected region.
    return BLOCK_RE.findall(content)


def find_violations(base_ref, head_ref="HEAD"):
    """Return docs paths whose generated regions differ between two refs."""
    base_sha = _resolve_ref(base_ref)
    head_sha = _resolve_ref(head_ref)
    violations = []
    for old_path, new_path in _changed_docs(base_sha, head_sha):
        old_blocks = _generated_blocks(_read_file(base_sha, old_path))
        new_blocks = _generated_blocks(_read_file(head_sha, new_path))
        if old_blocks != new_blocks:
            violations.append(new_path or old_path)
    return sorted(set(violations))


def check_autogenerated_regions(base_ref, head_ref="HEAD"):
    """Return True when generated regions are unchanged, else False."""
    try:
        violations = find_violations(base_ref, head_ref)
    except RuntimeError as error:
        print(f"Error: cannot check AUTOGENERATED regions: {error}")
        return False

    if not violations:
        print("OK: no direct edits inside AUTOGENERATED regions.")
        return True

    print("Direct edits inside AUTOGENERATED regions detected:\n")
    print("\n".join(f"  {path}" for path in violations))
    print(
        "\nEdit the generator's source data instead. Intentional generated-docs "
        "updates must carry the 'pr-autogenerated-docs' label."
    )
    return False


def main(argv=None):
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("base_ref", help="Git revision to compare against")
    parser.add_argument("--head-ref", default="HEAD", help="Git revision to check")
    args = parser.parse_args(argv)
    return 0 if check_autogenerated_regions(args.base_ref, args.head_ref) else 1


if __name__ == "__main__":
    sys.exit(main())