datastore/ci/jobs/changelog_nightly.py

"""Nightly job: prepare `CHANGELOG.md` for the upcoming release.

Maintains one bot branch and one draft PR per release cycle
(`auto/changelog-X.Y`, "Prepare changelog for X.Y"). Every day the job:

1. Generates raw changelog entries for the pull requests merged into `master`
   since the previous run, using `tests/ci/changelog.py` inside the
   `clickhouse/style-test` container (the same way `create_release.yml` runs
   it). The raw entries are inserted into `CHANGELOG.md` as a block delimited
   by HTML-comment markers, above the first released section, and committed.
2. Edits the raw block following `.claude/skills/edit-changelog/SKILL.md`
   (via the `codex` CLI, like the Code Review job) and integrates the result
   into the in-progress release section, as a separate commit. The
   generate/edit split keeps both states reviewable, mirroring the manual
   autogenerated-then-edited commit pairs of past releases.

The point up to which the changelog was generated is remembered as a
`Changelog-generated-up-to: <sha>` trailer in the generate commit message, so
the branch itself carries the state and the PR diff stays clean. The first
run of a cycle starts from the `vX.Y.1.1-new` tag that marks the cycle start
on `master`.

When `master` moves to the next version (the release branch is cut), the job
gives the previous cycle's branch a final catch-up (up to the new
`v*.1.1-new` tag) as long as its PR is still open, and starts a fresh
branch/PR for the new version. The draft PR is finalized manually by the
release manager (release date, presentation/video links, last-minute edits).

Fail-closed: a failed generation never commits; a failed edit resets the
working tree to the generate commit (which is still pushed, so the raw
entries stay visible in the PR and are picked up by the next run) and fails
the job. If a human closed the PR without merging, the branch is left alone.
"""

import os
import re
import shlex
import subprocess
import tempfile
import time
import traceback

from ci.praktika import Secret
from ci.praktika.gh_auth import GHAuth
from ci.praktika.info import Info
from ci.praktika.result import Result
from ci.praktika.utils import Shell

CHANGELOG_FILE = "CHANGELOG.md"
BRANCH_PREFIX = "auto/changelog-"
RAW_BEGIN = "<!-- CHANGELOG-RAW-BEGIN: auto-generated entries below are edited and removed by the NightlyChangelog CI job; do not edit them manually -->"
RAW_BEGIN_PREFIX = "<!-- CHANGELOG-RAW-BEGIN"
RAW_END = "<!-- CHANGELOG-RAW-END -->"
STATE_TRAILER = "Changelog-generated-up-to:"
RAW_OUTPUT = "./ci/tmp/changelog_raw_delta.md"
EDIT_SKILL = ".claude/skills/edit-changelog/SKILL.md"
STYLE_TEST_IMAGE = "clickhouse/style-test:latest"

# OpenAI API key for the codex CLI, same secret as the Code Review job.
OPENAI_KEY_SECRET = "/ci/llm/openai_api_key"
MAX_EDIT_ATTEMPTS = 3

VERSIONS_FILE = "cmake/autogenerated_versions.txt"


def _anchor_id(version):
    """'26.7' -> '267', the anchor used in CHANGELOG.md section headers."""
    return version.replace(".", "")


def _branch(version):
    return f"{BRANCH_PREFIX}{version}"


def _start_tag(version):
    return f"v{version}.1.1-new"


def _sha(ref):
    return Shell.get_output(f"git rev-parse '{ref}^{{}}'", strict=True).strip()


def get_master_version():
    """Version 'X.Y' currently developed on master = the upcoming release."""
    text = Shell.get_output(f"git show origin/master:{VERSIONS_FILE}", strict=True)
    major = re.search(r"SET\(VERSION_MAJOR (\d+)\)", text)
    minor = re.search(r"SET\(VERSION_MINOR (\d+)\)", text)
    assert major and minor, f"Cannot parse version from {VERSIONS_FILE}"
    return f"{major.group(1)}.{minor.group(1)}"


def get_cycle_start_tags():
    """All vX.Y.1.1-new tags (each marks the start of a development cycle)."""
    tags = {}
    for tag in Shell.get_output("git tag --list 'v*-new'").splitlines():
        m = re.fullmatch(r"v(\d+)\.(\d+)\.1\.1-new", tag.strip())
        if m:
            tags[(int(m.group(1)), int(m.group(2)))] = tag.strip()
    return tags


def get_cycle_end_ref(version, master_version):
    """The ref up to which the changelog of `version` should be generated.

    For the version currently developed on master it is the master head.
    For a past version it is the tag that started the next cycle, i.e. the
    point where master moved on to the next version.
    """
    if version == master_version:
        return "origin/master"
    tags = get_cycle_start_tags()
    key = tuple(int(x) for x in version.split("."))
    later = sorted(k for k in tags if k > key)
    if not later:
        return None
    return tags[later[0]]


def get_last_generated_sha(branch):
    """Read the state trailer from the branch-only commits, newest first.
    Requires origin/{branch} to be fetched (checkout_branch does that)."""
    log = Shell.get_output(
        f"git log --format=%B origin/master..origin/{branch}", strict=True
    )
    for line in log.splitlines():
        m = re.fullmatch(rf"{STATE_TRAILER}\s*([0-9a-f]{{40}})", line.strip())
        if m:
            return m.group(1)
    return None


def determine_from_ref(version, branch, exists_on_remote):
    """The point to continue the generation from: the last state trailer of
    the branch, or the cycle start tag for a fresh branch. Must run after
    checkout_branch. Refuses the tag fallback when the branch already carries
    changelog content: regenerating from the cycle start would duplicate the
    entries that were already integrated."""
    if exists_on_remote:
        from_ref = get_last_generated_sha(branch)
        if from_ref:
            return from_ref
        text = _read_changelog()
        if (
            RAW_BEGIN_PREFIX in text
            or extract_in_progress_section(text, _anchor_id(version)) is not None
        ):
            raise RuntimeError(
                f"Branch {branch} contains changelog content for {version} but "
                f"no '{STATE_TRAILER}' trailer in "
                f"origin/master..origin/{branch}; refusing to regenerate from "
                f"the cycle start, manual investigation required"
            )
    return _start_tag(version)


def remote_branch_exists(branch):
    return bool(Shell.get_output(f"git ls-remote --heads origin {branch}").strip())


def get_pr_states(branch):
    """States of the PRs from the repository's own branch `branch`. Fork PRs
    that happen to use the same head branch name are excluded: they must not
    make the job think its own PR already exists or was closed."""
    out = Shell.get_output(
        f"gh pr list --head {branch} --base master --state all "
        f"--repo {shlex.quote(Info().repo_name)} "
        "--json number,state,isCrossRepository "
        "--jq '.[] | select(.isCrossRepository | not) | \"\\(.number) \\(.state)\"'",
        strict=True,
    )
    return [tuple(line.split()) for line in out.splitlines() if line.strip()]


# --- CHANGELOG.md text manipulation ----------------------------------------

RELEASE_HEADING_RE = re.compile(r'^### <a id="(\d+)"></a>')


def strip_generated_preamble(raw_text):
    """Drop the docs frontmatter and the year header emitted by changelog.py,
    keeping from the '### ClickHouse release ...' line on."""
    lines = raw_text.splitlines()
    for i, line in enumerate(lines):
        if line.startswith("### "):
            return "\n".join(lines[i:]).strip()
    raise ValueError("No '### ' release header in the generated changelog")


def count_entries(text):
    return len(re.findall(r"^\* ", text, flags=re.MULTILINE))


def make_raw_block(body):
    return f"{RAW_BEGIN}\n{body}\n{RAW_END}"


def insert_before_first_release(text, block):
    """Insert `block` right above the first release section heading."""
    lines = text.splitlines()
    for i, line in enumerate(lines):
        if RELEASE_HEADING_RE.match(line):
            return "\n".join(lines[:i] + block.splitlines() + [""] + lines[i:]) + "\n"
    raise ValueError(f"No release section heading found in {CHANGELOG_FILE}")


def split_at_first_released_section(text, anchor):
    """Split into (head, tail) where tail starts at the first release heading
    that is not the in-progress section `anchor`. The tail must never be
    touched by the edit step."""
    lines = text.splitlines()
    for i, line in enumerate(lines):
        m = RELEASE_HEADING_RE.match(line)
        if m and m.group(1) != anchor:
            return "\n".join(lines[:i]), "\n".join(lines[i:])
    return text, ""


def extract_in_progress_section(text, anchor):
    lines = text.splitlines()
    start = None
    for i, line in enumerate(lines):
        m = RELEASE_HEADING_RE.match(line)
        if m and m.group(1) == anchor and start is None:
            start = i
        elif m and start is not None:
            return "\n".join(lines[start:i]).rstrip()
    if start is not None:
        return "\n".join(lines[start:]).rstrip()
    return None


def extract_raw_blocks(text):
    blocks = []
    lines = text.splitlines()
    begin = None
    for i, line in enumerate(lines):
        if line.startswith(RAW_BEGIN_PREFIX):
            begin = i
        elif line.strip() == RAW_END and begin is not None:
            blocks.append("\n".join(lines[begin : i + 1]))
            begin = None
    return blocks


def extract_toc_line(text, version):
    for line in text.splitlines():
        if line.startswith(f"**[ClickHouse release v{version},"):
            return line
    return None


def extract_pr_links(text):
    """PR numbers referenced as changelog entry links in `text`."""
    return set(
        re.findall(
            r"\[#(\d+)\]\(https://github\.com/[^/)]+/[^/)]+/pull/\d+\)", text
        )
    )


# The attribution form the generator emits at the end of every entry:
# `[#N](.../pull/N) ([Author](...))`. Inline mentions of other PRs inside the
# entry text lack the author parenthesis.
PR_ATTRIBUTION_RE = re.compile(
    r"\[#(\d+)\]\(https://github\.com/[^/)]+/[^/)]+/pull/\d+\) \(\["
)

# Categories whose raw entries the editing agent must carry into the edited
# section (possibly rewritten, merged, or under another category — the PR
# link must survive). The skill legitimately prunes the other sections
# (NOT FOR CHANGELOG, NO CL ENTRY, Build/Testing/Packaging).
STRICT_RETENTION_CATEGORIES = {
    "Backward Incompatible Change",
    "New Feature",
    "Experimental Feature",
    "Performance Improvement",
    "Improvement",
    "Bug Fix (user-visible misbehavior in an official stable release)",
}


# A bullet that IS a revert: the entry text (the generator uses the PR title
# for entries without a changelog description) starts with "Revert"/"Reverts"
# — possibly wrapped in the NO CL ENTRY prefix — or quotes a git revert
# commit message. A bullet merely mentioning reverts ("Improve revert
# logic ...") earns no deletion credit.
_REVERT_ENTRY_RE = re.compile(
    r"""(?ix)^\*\s* (?:NO\ CL\ ENTRY:\s*'?\s*)? revert(s|ed)?\b"""
)


def raw_strict_prs_and_reverts(text):
    """From the raw marker blocks in `text`: the attribution PR numbers of
    entries under the strict-retention categories, and the PR numbers of the
    revert entries (each licenses the deletion of the entry it reverts)."""
    strict_prs = set()
    revert_prs = []
    for block in extract_raw_blocks(text):
        category = None
        for line in block.splitlines():
            if line.startswith("#### "):
                category = line[5:].strip()
            elif line.startswith("* "):
                if _REVERT_ENTRY_RE.match(line) or "this reverts commit" in line.lower():
                    revert_prs.extend(PR_ATTRIBUTION_RE.findall(line))
                if category in STRICT_RETENTION_CATEGORIES:
                    strict_prs.update(PR_ATTRIBUTION_RE.findall(line))
    return strict_prs, revert_prs


def resolve_reverted_originals(revert_prs):
    """Map revert PRs to the PRs they revert, via the `Reverts owner/repo#N`
    line GitHub puts into revert PR bodies. Returns (originals, unresolved):
    unresolved reverts (a manual revert without the marker, or a failed
    lookup) grant no deletion credit — the caller fails closed on any
    disappearance they cannot bind to a resolved original."""
    originals = set()
    unresolved = []
    for pr in revert_prs:
        body = Shell.get_output(
            f"gh pr view {pr} --repo {shlex.quote(Info().repo_name)} "
            "--json body --jq .body"
        )
        found = re.findall(r"(?i)reverts\s+[\w.-]+/[\w.-]+#(\d+)", body)
        if found:
            originals.update(found)
        else:
            unresolved.append(pr)
    return originals, unresolved


def compose_on_master(master_text, version, toc_line, raw_blocks, section):
    """Rebuild our additions on top of master's CHANGELOG.md. Used to resolve
    the merge conflict that appears when master's CHANGELOG.md changed (e.g.
    the previous release section landed while this branch was open)."""
    anchor = _anchor_id(version)
    if f'<a id="{anchor}"></a>' in master_text:
        raise ValueError(
            f"Master already contains a section for {version}; "
            "manual reconciliation required"
        )
    text = master_text
    if toc_line:
        lines = text.splitlines()
        for i, line in enumerate(lines):
            if line.startswith("**[ClickHouse release v"):
                lines.insert(i, toc_line)
                break
        else:
            raise ValueError("No table-of-contents entries found on master")
        text = "\n".join(lines) + ("\n" if text.endswith("\n") else "")
    for part in [section] + raw_blocks:
        if part:
            text = insert_before_first_release(text, part)
    return text


def _untracked_files():
    """Untracked, not-ignored files (the gitignored ci/tmp scratch space is
    excluded, so the job's own temporary files never show up here).
    NUL-delimited so file names with spaces survive."""
    out = Shell.get_output(
        "git ls-files -z --others --exclude-standard", strict=True
    )
    return {path for path in out.split("\0") if path}


def _reset_worktree(base_sha, pre_untracked=None):
    """Restore the branch and all tracked files to the given commit, and
    remove untracked files that appeared since `pre_untracked` was snapshot
    (fail-closed cleanup after a bad edit attempt, including one where the
    editing agent created commits or new files of its own)."""
    Shell.check(f"git reset --hard {base_sha}", verbose=True, strict=True)
    if pre_untracked is not None:
        for path in sorted(_untracked_files() - pre_untracked):
            print(f"Removing file left behind by the editing agent: {path}")
            os.unlink(path)


def _read_changelog():
    with open(CHANGELOG_FILE, encoding="utf-8") as fd:
        return fd.read()


def _write_changelog(text):
    with open(CHANGELOG_FILE, "w", encoding="utf-8") as fd:
        fd.write(text)


# --- steps ------------------------------------------------------------------
# Each step returns a human-readable info string on success and raises on
# failure; _step() adapts that to praktika's Result convention.


def prepare_repo():
    if Shell.get_output("git rev-parse --is-shallow-repository").strip() == "true":
        Shell.check("git fetch --unshallow origin", verbose=True, strict=True)
    # Explicit refspec: the CI checkout restricts remote.origin.fetch to the
    # triggering ref, so a plain `git fetch origin master` would not be
    # guaranteed to update the remote-tracking ref.
    Shell.check(
        "git fetch --tags origin '+refs/heads/master:refs/remotes/origin/master'",
        verbose=True,
        strict=True,
    )
    Shell.check(
        'git config user.name "robot-clickhouse" && '
        'git config user.email "[email protected]"',
        verbose=True,
        strict=True,
    )
    return "Fetched master and tags"


def checkout_branch(branch, exists_on_remote):
    if exists_on_remote:
        # Explicit refspec, see prepare_repo.
        Shell.check(
            f"git fetch origin "
            f"'+refs/heads/{branch}:refs/remotes/origin/{branch}'",
            verbose=True,
            strict=True,
        )
        Shell.check(
            f"git checkout -B {branch} origin/{branch}", verbose=True, strict=True
        )
    else:
        Shell.check(
            f"git checkout -B {branch} origin/master", verbose=True, strict=True
        )


def reconcile_with_master(version):
    """Merge origin/master into the branch when master's CHANGELOG.md changed
    since the branch base. A conflict in CHANGELOG.md is resolved structurally
    by re-inserting our additions on top of master's file; any other conflict
    fails the job for manual resolution."""
    base = Shell.get_output("git merge-base HEAD origin/master", strict=True).strip()
    if Shell.check(f"git diff --quiet {base} origin/master -- {CHANGELOG_FILE}"):
        return "No CHANGELOG.md changes on master to reconcile"
    ours = _read_changelog()
    if Shell.check("git merge --no-edit origin/master", verbose=True):
        return "Merged origin/master cleanly"
    conflicts = Shell.get_output(
        "git diff --name-only --diff-filter=U", strict=True
    ).split()
    if conflicts != [CHANGELOG_FILE]:
        Shell.check("git merge --abort", verbose=True)
        raise RuntimeError(
            f"Merge of origin/master conflicts outside {CHANGELOG_FILE}: "
            f"{conflicts}; manual reconciliation required"
        )
    try:
        rebuilt = compose_on_master(
            Shell.get_output(
                f"git show origin/master:{CHANGELOG_FILE}", strict=True
            ),
            version,
            extract_toc_line(ours, version),
            extract_raw_blocks(ours),
            extract_in_progress_section(ours, _anchor_id(version)),
        )
        _write_changelog(rebuilt)
        Shell.check(f"git add {CHANGELOG_FILE}", strict=True)
        Shell.check("git commit --no-edit", verbose=True, strict=True)
    except BaseException:
        # Never leave the repository mid-merge: it would break the checkout
        # of the next processed cycle in the same run.
        Shell.check("git merge --abort", verbose=True)
        raise
    return "Merged origin/master, re-inserted the in-progress section on top"


def generate_raw_entries(version, from_ref, to_sha):
    """Run changelog.py in the style-test container for from_ref..to_sha and
    commit the raw entries as a marker block."""
    from_sha = _sha(from_ref)
    if from_sha == to_sha:
        return "No new commits since the last generation"
    if not Shell.check(f"git merge-base --is-ancestor {from_sha} {to_sha}"):
        raise RuntimeError(
            f"Recorded generation point {from_ref} ({from_sha}) is not an "
            f"ancestor of {to_sha}; refusing to generate"
        )

    if os.path.exists(RAW_OUTPUT):
        os.unlink(RAW_OUTPUT)
    Shell.check(f"docker pull {STYLE_TEST_IMAGE}", verbose=True, strict=True)
    # The token is passed through the environment (the command line, which
    # gets logged, only carries the unexpanded expressions): the host shell
    # exports it to the docker CLI and the in-container shell expands it for
    # changelog.py's --gh-user-or-token.
    cmd = (
        f"./tests/ci/changelog.py -v --jobs=5 "
        f'--gh-user-or-token "$GH_TOKEN" '
        f"--from {shlex.quote(from_ref)} --output {shlex.quote(RAW_OUTPUT)} {to_sha}"
    )
    if not Shell.check(
        f'GH_TOKEN="$(gh auth token)" '
        f'docker run --rm -u "$(id -u):$(id -g)" -e PYTHONUNBUFFERED=1 -e CI=1 '
        f"-e GH_TOKEN --network=host --volume=$(pwd):/wd --workdir=/wd "
        f"{STYLE_TEST_IMAGE} sh -c {shlex.quote(cmd)}",
        verbose=True,
    ):
        raise RuntimeError("changelog.py failed")

    with open(RAW_OUTPUT, encoding="utf-8") as fd:
        body = strip_generated_preamble(fd.read())
    entries = count_entries(body)
    from_short, to_short = from_sha[:11], to_sha[:11]
    if entries == 0:
        return f"No new changelog entries in {from_short}..{to_short}"

    _write_changelog(
        insert_before_first_release(_read_changelog(), make_raw_block(body))
    )
    message = (
        f"Update changelog for {version}: generate raw entries "
        f"({from_short}..{to_short})\n\n"
        f"{entries} raw entries generated with tests/ci/changelog.py "
        f"by the NightlyChangelog CI job.\n\n"
        f"{STATE_TRAILER} {to_sha}\n"
    )
    Shell.check(f"git add {CHANGELOG_FILE}", strict=True)
    Shell.check(f"git commit -m {shlex.quote(message)}", verbose=True, strict=True)
    return f"Generated {entries} raw entries for {from_short}..{to_short}"


def _edit_prompt(version):
    anchor = _anchor_id(version)
    return f"""\
You are updating {CHANGELOG_FILE} for the upcoming ClickHouse release {version} as part of an automated daily CI job.

{CHANGELOG_FILE} contains one or more blocks delimited by marker lines starting with `{RAW_BEGIN_PREFIX}` and ending with `{RAW_END}`. Each block holds raw auto-generated changelog entries (the output of `utils/changelog/changelog.py`) for pull requests merged into master since the previous run of this job.

Task:
1. Read {EDIT_SKILL} and apply its editing rules to the raw entries inside the marker blocks.
2. Integrate the edited entries into the in-progress release section, the one with the header containing `<a id="{anchor}"></a>`:
   - If that section does not exist yet, create it directly above the first released section, with exactly this header: `### <a id="{anchor}"></a> ClickHouse release {version}, FIXME (in progress)`, and add this line at the top of the "Table of Contents" list: `**[ClickHouse release v{version}, FIXME](#{anchor})**<br/>`.
   - Place entries under the proper `#### <Category>` headers in the preferred category order from the skill; create missing category headers as needed; keep exactly one blank line before each `####` header.
   - Entries already present in the in-progress section were edited on previous days: do not rewrite or reorder them, except where the skill requires it — a revert arriving in a raw block may delete or amend a previously added entry (skill section 2), and a follow-up PR may be merged into an existing bullet (skill section 7).
3. Delete the marker blocks, including the marker lines themselves, after integrating them.
4. Do not commit. Do not modify any file other than {CHANGELOG_FILE}. Do not touch the sections of already-released versions or anything below them. Keep the `FIXME` placeholders in the header and table of contents — the release manager fills in the date and links at release time.

The `gh` CLI is authenticated; use `gh pr view <N> --json title,body` when the skill requires consulting a pull request (reverts, unclear entries).
"""


def verify_edit(version, base_sha, pre_untracked=frozenset()):
    """Fail-closed checks after the editing agent ran, all against the
    pre-edit generate commit `base_sha` rather than a mutable HEAD (the agent
    has git access, and an agent-made commit must not bypass the checks).
    Returns an error string or None."""
    if _sha("HEAD") != base_sha:
        return "The editing agent created commits instead of leaving the edit uncommitted"
    changed = Shell.get_output(f"git diff --name-only {base_sha}", strict=True).split()
    if changed != [CHANGELOG_FILE]:
        return f"Unexpected working tree changes after edit: {changed or 'none'}"
    created = _untracked_files() - pre_untracked
    if created:
        return f"The editing agent created files: {sorted(created)}"
    text = _read_changelog()
    anchor = _anchor_id(version)
    if RAW_BEGIN_PREFIX in text or RAW_END in text:
        return "Raw entry markers are still present after edit"
    if f'<a id="{anchor}"></a>' not in text:
        return f"In-progress section for {version} is missing after edit"
    if not extract_toc_line(text, version):
        return f"Table-of-contents line for {version} is missing after edit"
    old_text = Shell.get_output(f"git show {base_sha}:{CHANGELOG_FILE}", strict=True)
    section = extract_in_progress_section(text, anchor)
    old_section = extract_in_progress_section(old_text, anchor) or ""
    if section is None:
        return "In-progress section disappeared after edit"
    # Entries must survive the edit: previously accumulated entries (the
    # skill only deletes one when a newly arrived revert cancels it, section
    # 2; merges keep all PR links, section 7), and today's raw entries from
    # the strict-retention categories (the skill prunes only the junk
    # sections; real entries are edited, merged, or moved — their PR links
    # stay, otherwise the entry is silently lost forever because the state
    # trailer has already advanced past it). A revert usually names the
    # original PR only in its title, so per-entry attribution is not
    # checkable — instead each revert entry in the raw block justifies at
    # most one loss, and a raw block without reverts justifies none.
    removed = extract_pr_links(old_section) - extract_pr_links(section)
    strict_prs, revert_prs = raw_strict_prs_and_reverts(old_text)
    # Membership is checked against the whole file: an entry may
    # legitimately already sit in a previously released section (a PR merged
    # into the last release branch after the cycle start appears in the
    # range but was already published).
    lost_new = {pr for pr in strict_prs if f"/pull/{pr})" not in text}
    disappeared = removed | lost_new
    if disappeared:
        originals, unresolved = resolve_reverted_originals(revert_prs)
        unmatched = disappeared - originals
        if unmatched:
            return (
                f"Entries disappeared in the edit without a matching revert "
                f"({sorted(unmatched)}); the raw block's reverts resolve to "
                f"{sorted(originals) or 'nothing'}"
                + (
                    f", and reverts {unresolved} could not be resolved "
                    f"(no `Reverts owner/repo#N` marker or failed lookup)"
                    if unresolved
                    else ""
                )
            )
    _, old_tail = split_at_first_released_section(old_text, anchor)
    _, new_tail = split_at_first_released_section(text, anchor)
    if old_tail.rstrip() != new_tail.rstrip():
        return "Sections of already-released versions were modified by the edit"
    return None


def edit_raw_entries(version):
    """Run the editing agent on the committed raw blocks and commit the result.
    Retries a few times; on failure resets to the generate commit and raises."""
    prompt = _edit_prompt(version)
    openai_key = Secret.Config(
        name=OPENAI_KEY_SECRET, type=Secret.Type.AWS_SSM_PARAMETER
    ).get_value()

    # The generate commit the edit must be based on; every verification and
    # cleanup is pinned to this SHA, not to HEAD, which the agent can move.
    base_sha = _sha("HEAD")
    pre_untracked = _untracked_files()
    last_error = None
    for attempt in range(1, MAX_EDIT_ATTEMPTS + 1):
        # Full reset to the generate commit: a failed attempt may have left
        # changes beyond CHANGELOG.md, new files, or even commits (all are
        # verify_edit failure modes), and they must not leak into the next
        # attempt or the next processed version.
        _reset_worktree(base_sha, pre_untracked)
        # The GitHub App token expires after an hour and an editing attempt
        # can run long; refresh so the agent's `gh pr view` calls work.
        GHAuth.auth_from_settings()
        try:
            with tempfile.TemporaryDirectory(dir="./ci/tmp") as codex_home:
                subprocess.run(
                    ["codex", "login", "--with-api-key"],
                    input=openai_key,
                    text=True,
                    check=True,
                    env={**os.environ, "CODEX_HOME": codex_home},
                )
                # Same invocation as the Code Review job (proven on these
                # runners): writable workspace, network for `gh pr view`.
                ok = Shell.check(
                    f"CODEX_HOME={shlex.quote(codex_home)} codex exec "
                    f"-m gpt-5.4 -c 'model_reasoning_effort=xhigh' "
                    f"-s workspace-write "
                    f"-c sandbox_workspace_write.network_access=true "
                    f"-c approval_policy=never "
                    f"--color never "
                    f"{shlex.quote(prompt)}",
                    verbose=True,
                )
            last_error = None if ok else "codex exited with non-zero status"
        except Exception as e:
            last_error = f"{type(e).__name__}: {e}"
            traceback.print_exc()
        if not last_error:
            last_error = verify_edit(version, base_sha, pre_untracked)
        if not last_error:
            break
        print(
            f"WARNING: edit attempt {attempt}/{MAX_EDIT_ATTEMPTS} failed: {last_error}"
        )
        if attempt < MAX_EDIT_ATTEMPTS:
            time.sleep(min(2**attempt, 60))

    if last_error:
        _reset_worktree(base_sha, pre_untracked)
        raise RuntimeError(
            f"Editing failed after {MAX_EDIT_ATTEMPTS} attempts: {last_error}. "
            f"The raw entries stay committed and will be edited by the next run."
        )

    message = (
        f"Update changelog for {version}: edit new entries\n\n"
        f"Edited by the NightlyChangelog CI job following {EDIT_SKILL}.\n"
    )
    Shell.check(f"git add {CHANGELOG_FILE}", strict=True)
    Shell.check(f"git commit -m {shlex.quote(message)}", verbose=True, strict=True)
    return "Edited the raw entries and committed the result"


def push_branch(branch, exists_on_remote):
    ahead_of = f"origin/{branch}" if exists_on_remote else "origin/master"
    if not Shell.get_output(f"git log --oneline {ahead_of}..HEAD").strip():
        return "Nothing to push"
    # Push with the App token rather than the GITHUB_TOKEN that the checkout
    # action persists in the http extraheader: GITHUB_TOKEN pushes do not
    # trigger workflow runs, so the PR would get no CI on refreshes. Same
    # pattern as the workflow-regeneration push in praktika's native_jobs.
    # The token expands at run time and is kept out of the logs
    # (verbose=False); the inherited extraheader is cleared so the tokenized
    # URL is the one that authenticates.
    repo_url = (
        "https://x-access-token:${token}@github.com/"
        + shlex.quote(Info().repo_name)
        + ".git"
    )
    refspec = shlex.quote(f"HEAD:refs/heads/{branch}")
    if not Shell.check(
        'token="$(gh auth token)" && '
        "git -c http.https://github.com/.extraheader= push "
        f"{repo_url} {refspec}",
        verbose=False,
    ):
        raise RuntimeError(f"Failed to push {branch}")
    return f"Pushed {branch}"


def ensure_pr(branch, version):
    # Refresh the App token: the preceding editing step may have outlived it.
    GHAuth.auth_from_settings()
    states = get_pr_states(branch)
    open_prs = [num for num, state in states if state == "OPEN"]
    if open_prs:
        return f"PR #{open_prs[0]} already open"
    if states:
        return f"PR was closed or merged ({states}); not creating a new one"
    body = f"""\
Automated daily preparation of `CHANGELOG.md` for the upcoming {version} release.

Every day the `NightlyChangelog` CI job appends the raw changelog entries for the pull requests newly merged into `master` (generated with `utils/changelog/changelog.py`) as one commit, and edits them following `.claude/skills/edit-changelog/SKILL.md` as a separate commit, so both the raw and the edited state stay reviewable. The point up to which entries were generated is recorded as a `{STATE_TRAILER}` trailer in the generate commits.

This pull request stays a draft until the release. The release manager finalizes it manually: fills in the release date and the presentation/video links (the `FIXME` placeholders), reviews the entries, and marks it ready.

### Changelog category (leave one):
- Not for changelog (changelog entry is not required)
"""
    with tempfile.NamedTemporaryFile(
        "w", dir="./ci/tmp", suffix=".md", delete=False
    ) as fd:
        fd.write(body)
        body_file = fd.name
    try:
        Shell.check(
            f"gh pr create --draft --base master --head {branch} "
            f"--repo {shlex.quote(Info().repo_name)} "
            f'--title {shlex.quote(f"Prepare changelog for {version}")} '
            f'--label "do not test" --body-file {shlex.quote(body_file)}',
            verbose=True,
            strict=True,
        )
    finally:
        os.unlink(body_file)
    return "Created a draft PR"


def cycle_skip_reason(branch, is_current, branch_exists, pr_states):
    """Decide whether a release cycle must not be processed: a finished cycle
    without a branch, a cycle whose PRs were all closed or merged by a human,
    or a branch that was deleted while its PRs exist (deleting the branch
    auto-closes the PR; silently recreating the branch from the cycle start
    would override that deliberate action)."""
    if not branch_exists and not is_current:
        return f"No branch {branch} for a finished cycle"
    if pr_states and not any(state == "OPEN" for _, state in pr_states):
        return f"PR for {branch} was closed or merged ({pr_states})"
    if not branch_exists and pr_states:
        return (
            f"Branch {branch} does not exist but its PRs do ({pr_states}); "
            f"refusing to recreate the branch, manual investigation required"
        )
    return None


def _step(results, name, fn):
    """Run `fn` as a praktika sub-result: `fn` returns an info string on
    success and raises on failure (stdout/traceback become the info then)."""
    holder = {}

    def _run():
        holder["info"] = fn()
        return True

    result = Result.from_commands_run(name=name, command=_run)
    if result.is_ok() and holder.get("info"):
        result.set_info(holder["info"])
        print(f"[{name}] {holder['info']}")
    results.append(result)
    return result.is_ok()


def process_version(version, master_version, results):
    """Run the full generate/edit/push/PR pipeline for one release cycle.
    Appends step Results to `results`; returns False if something failed."""
    branch = _branch(version)
    is_current = version == master_version

    def _skip(info):
        print(f"[{version}] {info}")
        results.append(
            Result(name=f"{version}: prepare", status=Result.Status.SKIPPED, info=info)
        )
        return True

    end_ref = get_cycle_end_ref(version, master_version)
    if end_ref is None:
        return _skip(f"No cycle end tag for finished cycle {version} yet")
    to_sha = _sha(end_ref)

    exists = remote_branch_exists(branch)
    skip_reason = cycle_skip_reason(
        branch, is_current, exists, get_pr_states(branch)
    )
    if skip_reason:
        return _skip(skip_reason)

    checkout_branch(branch, exists)
    from_ref = determine_from_ref(version, branch, exists)

    ok = True
    if exists:
        ok = _step(
            results,
            f"{version}: reconcile with master",
            lambda: reconcile_with_master(version),
        )

    if ok:
        ok = _step(
            results,
            f"{version}: generate raw entries",
            lambda: generate_raw_entries(version, from_ref, to_sha),
        )

    edit_ok = True
    if ok and RAW_BEGIN_PREFIX in _read_changelog():
        # A failed edit is reported, but the generate commit is still pushed
        # below so the raw entries are visible in the PR and picked up by the
        # next run.
        edit_ok = _step(
            results, f"{version}: edit entries", lambda: edit_raw_entries(version)
        )

    if ok:
        ok = _step(
            results, f"{version}: push", lambda: push_branch(branch, exists)
        )

    if ok and remote_branch_exists(branch):
        ok = _step(results, f"{version}: ensure PR", lambda: ensure_pr(branch, version))

    return ok and edit_ok


def main():
    # The GitHub App token from enable_gh_auth expires; re-issue it at job
    # start so the whole run (git push, gh, changelog.py) has a fresh one.
    GHAuth.auth_from_settings()
    os.makedirs("./ci/tmp", exist_ok=True)

    results = []
    ok = _step(results, "prepare repository", prepare_repo)

    if ok:
        master_version = get_master_version()
        print(f"Upcoming release on master: {master_version}")

        # Older cycles first (their final catch-up after the version bump),
        # then the current one.
        versions = []
        for ref in Shell.get_output(
            f"git ls-remote --heads origin '{BRANCH_PREFIX}*'"
        ).splitlines():
            m = re.search(rf"refs/heads/{re.escape(BRANCH_PREFIX)}(\d+\.\d+)$", ref)
            if m and m.group(1) != master_version:
                versions.append(m.group(1))
        versions = sorted(versions, key=lambda v: tuple(int(x) for x in v.split(".")))
        versions.append(master_version)

        for version in versions:
            try:
                process_version(version, master_version, results)
            except Exception as e:
                traceback.print_exc()
                results.append(
                    Result(
                        name=f"{version}: processing",
                        status=Result.Status.FAIL,
                        info=f"ERROR: {e}",
                    )
                )

    Result.create_from(results=results).complete_job()


if __name__ == "__main__":
    main()