datastore/ci/jobs/docs_autogen_nightly.py

import shlex
import sys

from ci.defs.defs import BASE_BRANCH
from praktika.git import Git
from praktika.info import Info
from praktika.result import Result
from praktika.utils import Shell

# The self-extracting master binary from the CH_ARM_BINARY artifact. Required
# artifacts are downloaded into the input dir under their basename, so it lands
# at ci/tmp/clickhouse (not the producer's build tree), as a plain file that
# must be made executable before use -- the same convention other consumers
# follow (e.g. ci/jobs/install_check.py). The generator resolves the path to an
# absolute one, so a repo-relative value is fine even though it runs the binary
# from a scratch directory.
BINARY = "ci/tmp/clickhouse"
GENERATOR = "ci/jobs/scripts/docs/autogenerate/autogenerate_docs.py"

# A stable branch, force-pushed each run, so the bot keeps a single open PR that
# always carries exactly one commit off the latest master.
BRANCH = "robot/docs-autogen"
# The label the docs check exempts from the generated-region guard (see
# ci/jobs/docs_job_mintlify.py). It also skips the changelog requirement.
# The label is provisioned once in the repository as part of the rollout (it
# already exists); the workflow only attaches it, so it stays the single source
# of truth for the label's color and description.
LABEL = "pr-autogenerated-docs"
TITLE = "Docs: regenerate reference documentation from source"
REVIEWERS = ("Blargian", "dhtclk")

BODY = """\
This pull request is opened automatically by the nightly documentation
autogeneration workflow. It regenerates settings, functions, table and database
engines, data types, formats, table functions, and window functions from the
structured documentation embedded in the Datastore source and exposed through
the corresponding `system.*` tables.

The generator preserves page frontmatter and hand-written content outside the
`{/*AUTOGENERATED_START*/}` / `{/*AUTOGENERATED_END*/}` regions. Some pages are
fully generated below their frontmatter. Do not edit generated content by hand --
edit the structured documentation in the defining source code instead; the next
nightly run regenerates the pages.

### Changelog category (leave one):
- Not for changelog (changelog entry is not required)

### Changelog entry (a [user-readable short description](https://github.com/ClickHouse/ClickHouse/blob/master/docs/changelog_entry_guidelines.md) of the changes that goes into CHANGELOG.md):
"""


def regenerate():
    """Regenerate the docs tree in place from the master binary.

    The artifact downloader materializes a plain file, so make it executable
    before invoking the generator."""
    return Shell.check(
        f"chmod +x {BINARY} && python3 {GENERATOR} --write --binary {BINARY} --docs-dir docs",
        verbose=True,
    )


def _open_bot_pr():
    """Return the number of the open bot pull request for BRANCH in the base
    repository, or "" if none.

    `gh pr list --head` matches by bare branch name, so a fork PR opened from a
    branch also named BRANCH would show up too. The same head branch can also
    have multiple open PRs when their base branches differ. Scope to the base
    branch and filter to non-cross-repository PRs so we only ever act on the
    bot's own PR to the intended base."""
    return Shell.get_output(
        f"gh pr list --head {BRANCH} --base {BASE_BRANCH} --state open"
        " --json number,isCrossRepository"
        " --jq 'map(select(.isCrossRepository == false)) | .[0].number // empty'"
    ).strip()


def _push_branch():
    """Force-push HEAD as the GitHub App so refreshed PRs trigger CI."""
    return Git.push(
        Info().repo_name,
        f"HEAD:refs/heads/{BRANCH}",
        force=True,
    )


def open_or_refresh_pr():
    """Commit the regenerated docs and open (or refresh) the bot's pull request.

    When regeneration produced no changes the docs are already in sync with the
    source, so any pull request the bot opened earlier is now stale (someone may
    have merged the same regeneration, or a later source change made the tree
    match master again). Rather than leave it advertising outdated changes, the
    stale pull request is closed and its branch deleted."""
    if not Shell.get_output("git status --porcelain docs/").strip():
        print("Docs already in sync with the source; no changes to publish.")
        pr = _open_bot_pr()
        if not pr:
            return True
        print(f"Closing stale bot pull request #{pr} and deleting branch {BRANCH}.")
        return Shell.check(
            f"gh pr close {pr} --delete-branch --comment "
            + shlex.quote(
                "The generated documentation is back in sync with the source; "
                "closing this stale autogenerated pull request."
            ),
            verbose=True,
        )

    commit = [
        "git config user.name 'robot-clickhouse'",
        "git config user.email '[email protected]'",
        f"git checkout -B {BRANCH}",
        "git add -A docs/",
        f"git commit -m {shlex.quote(TITLE)}",
    ]
    if not Shell.check(" && ".join(commit), verbose=True):
        return False

    if not _push_branch():
        return False

    pr = _open_bot_pr()
    if not pr:
        # First run: no bot PR yet -- create it with the label and request the
        # docs maintainers directly so the generated changes are reviewed.
        reviewers = " ".join(
            f"--reviewer {shlex.quote(reviewer)}" for reviewer in REVIEWERS
        )
        return Shell.check(
            f"gh pr create --base {BASE_BRANCH} --head {BRANCH} "
            f"--title {shlex.quote(TITLE)} --body {shlex.quote(BODY)} "
            f"--label {LABEL} {reviewers}",
            verbose=True,
        )
    # Existing PR: the force-push above already refreshed it. Ensure the label is
    # present, addressing the PR by number so a same-named fork PR is untouched.
    return Shell.check(f"gh pr edit {pr} --add-label {LABEL}", verbose=True)


def on_current_base_branch():
    """Refuse to run unless HEAD is the current tip of the base branch.

    The workflow is SCHEDULE-triggered, but GitHub also generates a
    `workflow_dispatch` trigger and reruns replay their original ref/sha. The
    branch-name check rejects feature-branch dispatches; fetching the remote base
    branch and comparing its tip with HEAD also rejects reruns of older `master`
    commits. The check runs again after generation so a run cannot publish if
    `master` advanced while it was working."""
    branch = Info().git_branch
    if branch != BASE_BRANCH:
        print(
            f"Refusing to run: this workflow force-pushes '{BRANCH}' and manages a"
            f" pull request, so it must run against '{BASE_BRANCH}', not '{branch}'"
            " (e.g. a manual workflow_dispatch or a rerun from another ref).",
            file=sys.stderr,
        )
        return False

    if not Shell.check(
        f"git fetch --no-tags origin {shlex.quote(BASE_BRANCH)}", verbose=True
    ):
        print(
            f"Refusing to run: could not fetch the current '{BASE_BRANCH}' tip.",
            file=sys.stderr,
        )
        return False

    head = Shell.get_output("git rev-parse HEAD").strip()
    current_base = Shell.get_output("git rev-parse FETCH_HEAD").strip()
    if not head or not current_base or head != current_base:
        print(
            f"Refusing to run: HEAD ({head or 'unknown'}) is not the current "
            f"'{BASE_BRANCH}' tip ({current_base or 'unknown'}).",
            file=sys.stderr,
        )
        return False
    return True


if __name__ == "__main__":
    results = [
        Result.from_commands_run(
            name="Run against current master", command=on_current_base_branch
        )
    ]
    # Fail closed: only regenerate / touch the pull request when the guard and
    # each prior step succeeded.
    if results[-1].is_ok():
        results.append(
            Result.from_commands_run(name="Regenerate docs", command=regenerate)
        )
    if results[-1].is_ok():
        results.append(
            Result.from_commands_run(
                name="Master is still current", command=on_current_base_branch
            )
        )
    if results[-1].is_ok():
        results.append(
            Result.from_commands_run(
                name="Open or refresh PR", command=open_or_refresh_pr
            )
        )
    Result.create_from(results=results).complete_job()