datastore/ci/jobs/cloud_api_docs_nightly.py

"""Nightly job: sync the Cloud API reference docs from the live OpenAPI spec.

Runs `generate_cloud_api_reference.py --write`, and if the generated pages or the
navigation drifted from the spec, opens (or refreshes) a single bot
pull request. The bot branch is stable so re-runs update one PR instead of
opening a new one every night.

Fail-closed: the PR is only touched when regeneration succeeded and produced a
real diff; a failed regeneration never pushes a branch or opens a PR.
"""

import os
import re
import shlex

from praktika.result import Result
from praktika.utils import Shell

GENERATOR = "./ci/jobs/scripts/docs/generate_cloud_api_reference.py"
BOT_BRANCH = "robot/cloud-api-docs"
REPOSITORY = "ClickHouse/ClickHouse"
# Paths the generator owns; everything staged into the bot PR lives here.
DOCS_PATHS = (
    "docs/products/cloud/api-reference docs/_specs/cloud-openapi.json "
    "docs/products/cloud/navigation.json"
)

PR_TITLE = "Docs: sync Cloud API reference from the OpenAPI spec"
PR_BODY = f"""\
Automated nightly sync of the ClickHouse Cloud API reference documentation from \
the live OpenAPI spec (`https://api.clickhouse.cloud/v1`).

This refreshes the pinned spec copy (`docs/_specs/cloud-openapi.json`, which the \
documentation pages resolve against), the badge stubs under \
`docs/products/cloud/api-reference/` (per-endpoint maturity badges and \
descriptions), and the "API reference" navigation fragment. Most nights nothing \
changes and no PR is opened; when the upstream API changes, this PR carries the \
diff and self-updates until merged.

Generated by `{GENERATOR}`.

### Changelog category (leave one):
- Documentation (changelog entry is not required)

### Changelog entry (a [user-readable short description]\
(https://github.com/ClickHouse/ClickHouse/blob/master/docs/changelog_entry_guidelines.md) \
of the changes that goes into CHANGELOG.md):
Sync the ClickHouse Cloud API reference documentation from the OpenAPI spec.
"""

TOKENIZED_GITHUB_URL = re.compile(r"(https://x-access-token:)[^@\s]+(@github\.com/)")


def regenerate():
    return Shell.check(f"python3 {GENERATOR} --write", verbose=True)


def has_changes():
    return bool(Shell.get_output(f"git status --porcelain -- {DOCS_PATHS}").strip())


def open_or_refresh_pr():
    repository = os.getenv("GITHUB_REPOSITORY", "")
    if repository != REPOSITORY:
        print(
            "ERROR: the Cloud API docs workflow must run only in "
            f"{REPOSITORY}, got {repository or '(unset)'}"
        )
        return False

    prepare = [
        'git config user.name "robot-clickhouse"',
        'git config user.email "[email protected]"',
        f"git checkout -B {BOT_BRANCH}",
        f"git add -A {DOCS_PATHS}",
        f"git commit -m {shlex.quote(PR_TITLE)}",
    ]
    if not Shell.check(" && ".join(prepare), verbose=True):
        return False

    # actions/checkout persists the workflow GITHUB_TOKEN as an HTTP extraheader,
    # so a plain `git push origin` ignores the GitHub App token authenticated by
    # enable_gh_auth and pushes as github-actions[bot]. Clear that extraheader and
    # use the App token explicitly. Keep the command out of logs because its URL
    # contains the expanded token.
    repo_url = (
        "https://x-access-token:${token}@github.com/"
        + shlex.quote(REPOSITORY)
        + ".git"
    )
    refspec = shlex.quote(f"{BOT_BRANCH}:refs/heads/{BOT_BRANCH}")
    push = (
        'token="$(gh auth token)" && '
        "git -c http.https://github.com/.extraheader= push -f "
        f"{repo_url} {refspec}"
    )
    return_code, stdout, stderr = Shell.get_res_stdout_stderr(push, verbose=False)
    if return_code != 0:
        output = "\n".join(part for part in (stdout, stderr) if part)
        output = TOKENIZED_GITHUB_URL.sub(r"\1***\2", output)
        print("ERROR: failed to push the Cloud API docs bot branch")
        if output:
            print(output)
        return False

    # A force-push already refreshed any open PR, so a PR is created only when
    # none exists. The existence probe is explicit rather than `gh pr create
    # || true`: that would swallow every creation failure (auth scopes, branch
    # permissions, body validation, GitHub outages) and let the job go green
    # while the drift never reaches reviewers.
    existing = Shell.get_output(
        f"gh pr list --head {BOT_BRANCH} --base master --state open "
        f"--repo {shlex.quote(REPOSITORY)} --json number --jq '.[].number'"
    ).strip()
    if existing:
        print(f"PR #{existing} already open; branch refreshed")
        return True
    return Shell.check(
        f"gh pr create --base master --head {BOT_BRANCH} "
        f"--repo {shlex.quote(REPOSITORY)} "
        f"--title {shlex.quote(PR_TITLE)} --body {shlex.quote(PR_BODY)}",
        verbose=True,
    )


if __name__ == "__main__":
    results = [Result.from_commands_run(name="Regenerate", command=regenerate)]

    if results[-1].is_ok():
        if has_changes():
            results.append(
                Result.from_commands_run(name="Open or refresh PR", command=open_or_refresh_pr)
            )
        else:
            results.append(
                Result.from_commands_run(name="Open or refresh PR", command=lambda: True)
            )
            results[-1].set_info("No changes; Cloud API docs already in sync")

    Result.create_from(results=results).complete_job()