datastore/ci/jobs/scripts/create_release.py

#!/usr/bin/env python3

"""
Prepare release machine:

### INSTALL PACKAGES
sudo apt update
sudo apt install --yes --no-install-recommends python3-dev python3-pip gh unzip
sudo apt install --yes python3-boto3
sudo apt install --yes s3fs

### INSTALL AWS CLI
cd /tmp
curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m).zip" -o "awscliv2.zip"
unzip awscliv2.zip
sudo ./aws/install
rm -rf aws*
cd -

### Install reprepro:
cd ~
sudo apt install dpkg-dev fakeroot libgpgme-dev libdb-dev libbz2-dev liblzma-dev libarchive-dev shunit2 db-util debhelper
git clone https://salsa.debian.org/debian/reprepro.git
cd reprepro
dpkg-buildpackage -b --no-sign && sudo dpkg -i ../reprepro_$(dpkg-parsechangelog --show-field Version)_$(dpkg-architecture -q DEB_HOST_ARCH).deb

### Install createrepo-c:
sudo apt install createrepo-c
createrepo_c --version
#Version: 0.17.3 (Features: DeltaRPM LegacyWeakdeps )

### Import gpg sign key
gpg --import key.pgp
gpg --list-secret-keys
"""

import argparse
import atexit
import dataclasses
import json
import os
import re
import shlex
import shutil
import sys
import tempfile
from contextlib import contextmanager
from copy import copy
from pathlib import Path
from typing import Iterator, List

from ci.jobs.scripts.clickhouse_version import (
    FILE_WITH_VERSION_PATH,
    CHVersion,
    VersionType,
)
from ci.praktika.gh import GH
from ci.praktika.git import Git
from ci.praktika.utils import Shell

# S3Helper requires boto3 (installed on release machines); ssh has no external deps.
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../../tests/ci"))
from s3_helper import S3Helper  # noqa: E402
from ssh import SSHAgent  # noqa: E402

GITHUB_REPOSITORY = os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse")
S3_BUILDS_BUCKET = os.getenv("S3_BUILDS_BUCKET", "clickhouse-builds")

GENERATED_CONTRIBUTORS = "./src/Storages/System/StorageSystemContributors.generated.cpp"
RELEASE_INFO_FILE = "/tmp/release_info.json"

with tempfile.NamedTemporaryFile("w", delete=False) as _known_hosts_file:
    _GIT_KNOWN_HOSTS_FILE = _known_hosts_file.name
    atexit.register(os.remove, _known_hosts_file.name)

ROBOT_GIT_USER_NAME = "robot-clickhouse"
ROBOT_GIT_USER_EMAIL = "[email protected]"

GIT_PREFIX = (
    f"git -c user.email={ROBOT_GIT_USER_EMAIL} "
    f"-c user.name={ROBOT_GIT_USER_NAME} -c commit.gpgsign=false "
    f"-c core.sshCommand='ssh -o UserKnownHostsFile={_GIT_KNOWN_HOSTS_FILE} "
    "-o StrictHostKeyChecking=accept-new'"
)

CONTRIBUTORS_TEMPLATE = """\
// autogenerated by {executer}
const char * auto_contributors[] {{
{contributors}
    nullptr}};
"""


class Labels:
    RELEASE = "release"
    RELEASE_LTS = "release-lts"


def is_latest_release_branch(branch: str, repo: str = GITHUB_REPOSITORY) -> bool:
    """Whether `branch` is the latest release branch — i.e. the head branch of
    the most recently created `release`-labeled PR. Release-domain logic (knows
    the `release` label / branch convention), so it lives here rather than in
    the generic praktika GH helper.

    The `gh pr list` read is retried and a persistent failure raises: mistaking
    a transient GitHub error for "not the latest branch" would silently skip the
    floating `latest` docker tag for an otherwise successful release. A
    successful `--json` read always prints at least `[]`, so an empty result
    means the command failed; genuinely no release PR is an empty JSON array."""
    raw = GH.get_output_with_retries(
        f'gh pr list --label release --repo {shlex.quote(repo)} --search "sort:created"'
        f" -L1 --json headRefName"
    )
    if not raw:
        raise RuntimeError(
            f"gh pr list failed for release PRs in repo [{repo}] after retries; "
            f"refusing to treat a failed lookup as 'not the latest release branch'"
        )
    prs = json.loads(raw)
    latest_branch = prs[0]["headRefName"] if prs else ""
    print(
        f"Latest release branch [{latest_branch}], checking [{branch}]:"
        f" is_latest={latest_branch == branch}"
    )
    return latest_branch == branch


def update_contributors(raise_error: bool = False) -> None:
    if Git.is_shallow():
        msg = "The repository is shallow, refusing to update contributors"
        if raise_error:
            raise RuntimeError(msg)
        print(f"WARNING: {msg}")
        return

    shortlog = Git.get_shortlog()
    escaping = str.maketrans({"\\": "\\\\", '"': '\\"'})
    contributors = sorted(
        [c.split(maxsplit=1)[-1].translate(escaping) for c in shortlog.splitlines()]
    )
    contributors_lines = [f'    "{c}",' for c in contributors]
    executer = Path(__file__).relative_to(Path.cwd())
    content = CONTRIBUTORS_TEMPLATE.format(
        executer=executer, contributors="\n".join(contributors_lines)
    )
    Path(GENERATED_CONTRIBUTORS).write_text(content, encoding="utf-8")


class ReleaseProgress:
    STARTED = "started"
    DOWNLOAD_PACKAGES = "download packages"
    PUSH_RELEASE_TAG = "push release tag"
    PUSH_NEW_RELEASE_BRANCH = "push new release branch"
    BUMP_VERSION = "bump version"
    CREATE_GH_RELEASE = "create GH release"
    EXPORT_TGZ = "export TGZ packages"
    EXPORT_RPM = "export RPM packages"
    EXPORT_DEB = "export DEB packages"
    TEST_TGZ = "test TGZ packages"
    TEST_RPM = "test RPM packages"
    TEST_DEB = "test DEB packages"
    MERGE_CREATED_PRS = "merge created PRs"
    COMPLETED = "completed"


class ReleaseContextManager:
    def __init__(self, release_progress):
        self.release_progress = release_progress
        self.release_info = None

    def __enter__(self):
        if self.release_progress == ReleaseProgress.STARTED:
            self.release_info = ReleaseInfo(
                release_branch="NA",
                release_type="NA",
                commit_sha=args.ref,
                release_tag="NA",
                version="NA",
                codename="NA",
                release_progress=ReleaseProgress.STARTED,
                latest=False,
            ).dump()
        else:
            self.release_info = ReleaseInfo.from_file()
            assert self.release_info
            # Record the furthest step reached (informational only). Step
            # ordering within a run is enforced by the workflow's own step
            # gating, not by a per-step status flag.
            self.release_info.release_progress = self.release_progress
            self.release_info.dump()
        return self.release_info

    def __exit__(self, exc_type, exc_value, traceback):
        # Persist whatever the body mutated on release_info. A failed step is
        # reflected by the workflow job result, not by a status flag here, so we
        # only dump and do not suppress the exception.
        assert self.release_info
        self.release_info.dump()
        return False


@dataclasses.dataclass
class ReleaseInfo:
    version: str
    release_type: str
    release_tag: str
    release_branch: str
    commit_sha: str
    latest: bool
    codename: str
    # Whether this release is the latest on its branch (controls the floating
    # minor/major docker tags). `latest` above is whether the branch is the
    # latest release branch (additionally controls the `latest` docker tag).
    is_branch_release: bool = False
    # Whether this run creates a new release (tag/version-bump/changelog). False
    # when re-publishing artifacts for an already-released or out-of-order ref.
    create_new_release: bool = False
    changelog_pr: str = ""
    version_bump_pr: str = ""
    prs_merged: bool = False
    release_url: str = ""
    debian: str = ""
    rpm: str = ""
    tgz: str = ""
    docker: str = ""
    release_progress: str = ""

    def is_patch(self):
        return self.release_branch != "master"

    def is_new_release_branch(self):
        return self.release_branch == "master"

    @staticmethod
    def from_file() -> "ReleaseInfo":
        with open(RELEASE_INFO_FILE, "r", encoding="utf-8") as f:
            return ReleaseInfo(**json.load(f))

    def dump(self):
        print(f"Dump release info into [{RELEASE_INFO_FILE}]")
        with open(RELEASE_INFO_FILE, "w", encoding="utf-8") as f:
            print(json.dumps(dataclasses.asdict(self), indent=2), file=f)
        return self

    @staticmethod
    def _is_empty_patch_release(patch: int, tweak: int) -> bool:
        """
        Whether a patch release would be empty and must be refused.

        For a patch release the tweak equals the number of commits since the
        previous release tag (see `Git.tweak`), so `tweak == 1` means the only
        commit on top of the previous release is the automated post-release
        version bump — there is nothing to release (e.g. `v25.8.28.1-lts`).

        The exception is `patch == 1`: that is the first user-facing
        `stable`/`lts` release of a freshly cut branch. Its previous tag is the
        non-user-facing `vX.Y.1.1-new`, and the single automated
        `testing -> stable/lts` version-update commit also yields `tweak == 1`.
        That release is legitimate and must be allowed. The post-release bump
        always increments `patch`, so an already-published branch is always at
        `patch >= 2` on a rerun.
        """
        return tweak == 1 and patch != 1

    def prepare(
        self, commit_ref: str, release_type: str, dry_run: bool = False
    ) -> "ReleaseInfo":
        assert release_type in ("patch", "new")
        # `commit_ref` (the workflow `ref` input) is interpolated into git
        # commands run through a shell below; fail closed on anything that is
        # not a plain branch/tag/SHA so a ref with shell metacharacters cannot
        # inject commands into the privileged release job.
        assert re.fullmatch(r"[0-9A-Za-z._/-]+", commit_ref), (
            f"ref [{commit_ref}] contains unexpected characters; expected a "
            f"branch, tag, or commit SHA"
        )
        version = None
        release_branch = None
        release_tag = None
        latest_release = False
        codename = ""
        # Whether the branch has already moved to a newer release than this one
        # (set for patch releases from the branch-tip version file). A new
        # release cuts a fresh minor, so it is always the latest.
        newer_release_exists = False

        if release_type == "new":
            if commit_ref != "master":
                Shell.check(
                    f"git merge-base --is-ancestor {commit_ref} origin/master",
                    strict=True,
                    verbose=True,
                )
            with checkout(commit_ref):
                commit_sha = Git.get_commit_sha(commit_ref)
                git = Git()
                version = CHVersion.get_current_version()
                release_branch = f"{version.major}.{version.minor}"
                expected_prev_tag = f"v{version.major}.{version.minor}.1.1-new"
                version.bump_release().with_description(VersionType.NEW)
                assert (
                    git.latest_tag == expected_prev_tag
                ), f"BUG: latest tag [{git.latest_tag}], expected [{expected_prev_tag}]"
                release_tag = version.describe

        if release_type == "patch":
            with checkout(commit_ref):
                commit_sha = Git.get_commit_sha(commit_ref)
                version = CHVersion.get_current_version()
                codename = version.get_stable_release_type()
                version.with_description(codename)
                release_branch = f"{version.major}.{version.minor}"
                release_tag = version.describe
            Shell.check(
                f"git merge-base --is-ancestor {commit_ref} origin/{release_branch}",
                strict=True,
                verbose=True,
            )

            # The branch's current version, read from the version file at its tip
            # (not from release tags). If this commit's version is older than the
            # branch tip, the branch has already moved to a newer release, so this
            # ref is behind / superseded.
            with checkout(f"origin/{release_branch}"):
                branch_version = CHVersion.get_current_version()
            newer_release_exists = version.is_older(branch_version)

            if is_latest_release_branch(release_branch, repo=GITHUB_REPOSITORY):
                print("This is going to be the latest release!")
                latest_release = True

        assert (
            release_branch
            and commit_sha
            and release_tag
            and version
            and (codename in ("lts", "stable") or release_type == "new")
        ), (
            f"Check: {release_branch}, {commit_sha}, {release_tag}, {version}"
        )

        self.release_branch = release_branch
        self.commit_sha = commit_sha
        self.release_tag = release_tag
        self.version = version.string
        self.codename = codename
        self.release_progress = ReleaseProgress.STARTED
        self.latest = latest_release

        # Is the branch already developing a newer release than this one? This
        # release is the latest on its branch unless the branch tip is newer —
        # controls the floating minor/major Docker tags (recovering the current
        # release re-applies them; recovering a superseded one does not).
        self.is_branch_release = not newer_release_exists

        # The operation is decided from the ref and the branch's version:
        #   * ref is an existing release tag -> recovery: re-publish exactly that
        #     release (allowed even for a superseded one);
        #   * ref is a branch/commit that is older than the branch tip -> out of
        #     order. A branch ref can't reach this (its tip is never older than
        #     itself), so it is a raw SHA pointing behind the tip; refuse it;
        #   * ref is a branch/commit whose own release tag already exists at this
        #     commit -> recovery: a rerun. auto_releases dispatches ref=<commit_sha>
        #     and GitHub's "Re-run failed jobs" replays the matrix with that same
        #     SHA while AutoReleaseInfo is not recomputed, so the tag pushed on the
        #     first attempt already points here — degrade to recovery instead of
        #     trying to create (and re-merge) the release a second time;
        #   * otherwise -> create the next release.
        #
        # The stale-SHA and superseded-recovery cases both compute the same
        # existing release_tag; only the ref KIND distinguishes them, so the tag
        # ref is checked first and the newer-release guard runs before the rerun
        # case.
        #
        # release_job.py defers the patch branch version bump to the very last
        # step (after publishing), so a rerun after any failure runs while the
        # branch tip still equals the released commit: newer_release_exists is
        # False and this reaches the rerun-recovery branch. The branch only moves
        # ahead once the whole release has succeeded, when no rerun is pending.
        # That keeps reruns recoverable without consulting release tags here.
        if Git.tag_exists(commit_ref):
            recover = True
            assert release_tag == commit_ref, (
                f"ref [{commit_ref}] is a release tag but the version at its commit "
                f"describes [{release_tag}]; refusing to re-publish a different "
                f"release"
            )
        elif newer_release_exists:
            raise RuntimeError(
                f"Refusing out-of-order release [{release_tag}] from [{commit_ref}]: "
                f"branch [{release_branch}] is already on a newer release. Pass a "
                f"release tag to recover an existing release, or the branch to "
                f"release its next commit."
            )
        elif Git.tag_exists(release_tag):
            tagged_sha = Git.get_commit_sha(release_tag)
            if tagged_sha != commit_sha:
                # The version computed from the file at this ref describes a
                # release that already exists at a different commit. This is the
                # stale-version-file hazard for a branch ref: the post-release
                # version bump has not been applied to the branch, so the tip
                # still describes an already-published release. Fail closed with
                # an actionable message rather than assert or silently mint a
                # colliding tag. (Detecting the wider "computed release is below
                # the branch's latest" case would need to scan release tags,
                # which the release job deliberately does not do — see
                # 80c722e39ae.)
                raise RuntimeError(
                    f"release tag [{release_tag}] already exists at [{tagged_sha}] "
                    f"but this run targets [{commit_sha}]. The version file at "
                    f"[{commit_ref}] is stale (it still describes an "
                    f"already-published release); land the post-release version "
                    f"bump on the branch, then dispatch its tip, or pass a "
                    f"release tag to recover an existing release."
                )
            recover = True
        else:
            # Creating (not recovering): refuse an empty patch release, where the
            # only commit since the previous release is the automated version
            # bump (tweak == 1). The recovery branches above are exempt.
            if release_type == "patch" and self._is_empty_patch_release(
                version.patch, version.tweak
            ):
                raise RuntimeError(
                    f"Refusing to create an empty patch release [{release_tag}] "
                    f"from [{commit_ref}]: version [{version.string}] has tweak 1, "
                    f"so the only commit since the previous release is the "
                    f"automated version bump — there is nothing to release."
                )
            recover = False
        self.create_new_release = not recover
        self.release_type = release_type
        return self

    def push_release_tag(self, dry_run: bool) -> None:
        print(
            f"Create and push release tag [{self.release_tag}], commit [{self.commit_sha}]"
        )
        # Create the annotated tag and push it as the App. Retry: on a repo this
        # size GitHub's push-time "does this ref touch .github/workflows" check
        # can time out and fail closed with a spurious "remote rejected ...
        # `workflows` scope may be required"; the tag points at an existing
        # commit (no workflow changes), so retrying lets the check complete.
        Git.push_tag(
            GITHUB_REPOSITORY,
            self.release_tag,
            self.commit_sha,
            f"Release {self.release_tag}",
            user_name=ROBOT_GIT_USER_NAME,
            user_email=ROBOT_GIT_USER_EMAIL,
            dry_run=dry_run,
            retries=3,
        )

    @staticmethod
    def _create_gh_label(label: str, color_hex: str, dry_run: bool) -> None:
        cmd = f"gh api repos/{GITHUB_REPOSITORY}/labels -f name={label} -f color={color_hex}"
        res = Shell.check(cmd, dry_run=dry_run, verbose=True)
        if not res:
            print("WARNING: failed to create backport labels for the new branch")

    def push_new_release_branch(self, dry_run: bool) -> None:
        version = CHVersion.get_current_version()
        new_release_branch = self.release_branch
        version_after_release = copy(version)
        version_after_release.bump_release()
        assert version_after_release.string == self.version, (
            f"Unexpected current version in git, must precede [{self.version}] by one step, "
            f"actual [{version.string}]"
        )
        if dry_run:
            Shell.check(
                f"{GIT_PREFIX} branch -l | grep -q {new_release_branch} && git branch -d {new_release_branch}"
            )
        print(
            f"Create and push new release branch [{new_release_branch}], commit [{self.commit_sha}]"
        )
        # Cut the branch from the exact released commit, not ambient master:
        # the workflow may be dispatched with a SHA, and master can move between
        # release selection and this step, which would otherwise point the tag
        # and the branch at different commits.
        with checkout(self.commit_sha):
            with checkout_new(new_release_branch):
                Git.push(
                    GITHUB_REPOSITORY,
                    f"HEAD:refs/heads/{new_release_branch}",
                    dry_run=dry_run,
                    strict=True,
                    retries=3,  # transient workflow-scope timeout (see push_release_tag)
                )

        print("Create and push backport tags for new release branch")
        ReleaseInfo._create_gh_label(
            f"v{new_release_branch}-must-backport", "10dbed", dry_run=dry_run
        )
        ReleaseInfo._create_gh_label(
            f"v{new_release_branch}-affected", "c2bfff", dry_run=dry_run
        )

    def get_version_bump_branch(self):
        return f"bump_version_{self.version}"

    def update_version_and_contributors_list(self, dry_run: bool) -> None:
        with checkout(self.commit_sha):
            version = CHVersion.get_current_version()
            if self.release_type == "patch":
                assert version.string == self.version, (
                    f"BUG: version in release info does not match version in git commit, "
                    f"expected [{self.version}], got [{version.string}]"
                )
                version.bump_patch()
            else:
                version.tweak = 1
            version.with_description(version.get_stable_release_type())

        with checkout(self.release_branch):
            if self.release_type == "new":
                # A freshly cut branch keeps its committed VERSION_GITHASH rather
                # than repointing it at the release commit.
                version.githash = CHVersion.get_release_version().githash
            else:
                version.githash = Shell.get_output_or_raise("git rev-parse HEAD")
            version.write()
            update_contributors(raise_error=True)
            cmd_commit_version_upd = (
                f"{GIT_PREFIX} commit '{FILE_WITH_VERSION_PATH}' '{GENERATED_CONTRIBUTORS}' "
                f"-m 'Update autogenerated version to {self.version} and contributors'"
            )
            # `git diff --quiet` exit codes: 0 = no changes, 1 = changes, >1 = error.
            # On a rerun the files are already up to date (exit 0), so skip the
            # commit + push silently. Treat >1 as a hard failure.
            diff_rc = Shell.run(
                f"git diff --quiet HEAD -- '{FILE_WITH_VERSION_PATH}' '{GENERATED_CONTRIBUTORS}'"
            )
            assert diff_rc <= 1, f"`git diff` failed with exit code {diff_rc}"
            files_changed = dry_run or diff_rc == 1
            if not files_changed:
                print(
                    f"Version files already up to date on {self.release_branch} — skipping commit and push (rerun)"
                )
            if files_changed:
                Shell.check(
                    cmd_commit_version_upd, strict=True, dry_run=dry_run, verbose=True
                )
                Git.push(
                    GITHUB_REPOSITORY,
                    f"HEAD:refs/heads/{self.release_branch}",
                    dry_run=dry_run,
                    strict=True,
                    retries=3,  # transient workflow-scope timeout (see push_release_tag)
                )
            if dry_run:
                Shell.check(
                    f"{GIT_PREFIX} diff '{FILE_WITH_VERSION_PATH}' '{GENERATED_CONTRIBUTORS}'",
                    verbose=True,
                )
                Shell.check(
                    f"{GIT_PREFIX} checkout '{FILE_WITH_VERSION_PATH}' '{GENERATED_CONTRIBUTORS}'",
                    verbose=True,
                )

        if self.release_type == "new":
            release_type = version.get_stable_release_type()
            print("Update version on master branch")
            branch_upd = self.get_version_bump_branch()
            with checkout(self.commit_sha):
                version = CHVersion.get_current_version()
                version.bump_release()
                version.with_description(VersionType.TESTING)
            with checkout("master"):
                with checkout_new(branch_upd):
                    # The post-release bump on master must point VERSION_GITHASH
                    # at master's head, not back at the release commit.
                    version.githash = Shell.get_output_or_raise("git rev-parse HEAD")
                    version.write()
                    update_contributors(raise_error=True)
                    actor = os.getenv("GITHUB_ACTOR", "") or "me"
                    body = (
                        f"Automatic version bump after release {self.release_tag}\n"
                        "### Changelog category (leave one):\n"
                        "- Not for changelog (changelog entry is not required)\n"
                    )
                    Shell.check(
                        f"{GIT_PREFIX} commit '{FILE_WITH_VERSION_PATH}' '{GENERATED_CONTRIBUTORS}' "
                        f"-m 'Update autogenerated version to {self.version} and contributors'",
                        strict=True,
                        dry_run=dry_run,
                        verbose=True,
                    )
                    # Rerun-safe (mirrors the changelog PR path): a previous
                    # attempt may have already pushed this branch and opened the
                    # PR. Force-push the freshly rebuilt branch, and only open the
                    # PR if one does not already exist for it.
                    Git.push(
                        GITHUB_REPOSITORY,
                        f"HEAD:refs/heads/{branch_upd}",
                        force=True,
                        dry_run=dry_run,
                        strict=True,
                        retries=3,  # transient workflow-scope timeout (see push_release_tag)
                    )
                    existing_bump_pr = (
                        "" if dry_run
                        else GH.get_pr_url_by_branch(
                            branch=branch_upd, repo=GITHUB_REPOSITORY
                        )
                    )
                    if existing_bump_pr:
                        print(
                            f"Version bump PR already exists [{existing_bump_pr}]"
                            " - skipping create"
                        )
                    else:
                        Shell.check(
                            f"gh pr create --repo {GITHUB_REPOSITORY} --title 'Update version after release' "
                            f"--head {branch_upd} --base master --body \"{body}\" --assignee {actor}",
                            strict=True,
                            dry_run=dry_run,
                            verbose=True,
                        )
                    if dry_run:
                        Shell.check(
                            f"{GIT_PREFIX} diff '{FILE_WITH_VERSION_PATH}' '{GENERATED_CONTRIBUTORS}'",
                            verbose=True,
                        )
                        Shell.check(
                            f"{GIT_PREFIX} checkout '{FILE_WITH_VERSION_PATH}' '{GENERATED_CONTRIBUTORS}'",
                            verbose=True,
                        )
                        self.version_bump_pr = "dry-run"
                    else:
                        self.version_bump_pr = GH.get_pr_url_by_branch(
                            branch=branch_upd, repo=GITHUB_REPOSITORY
                        )

            print("Create Release PR")
            with checkout(self.release_branch):
                pr_labels = f"--label {Labels.RELEASE}"
                if release_type == VersionType.LTS:
                    pr_labels += f" --label {Labels.RELEASE_LTS}"
                # Rerun-safe (mirrors the version-bump PR path): a previous
                # attempt may have already opened the release PR for this branch.
                # Only open it if one does not already exist, so a rerun reuses it
                # instead of failing on a duplicate `gh pr create`.
                existing_release_pr = (
                    "" if dry_run
                    else GH.get_pr_url_by_branch(
                        branch=self.release_branch, repo=GITHUB_REPOSITORY
                    )
                )
                if existing_release_pr:
                    print(
                        f"Release PR already exists [{existing_release_pr}]"
                        " - skipping create"
                    )
                else:
                    Shell.check(
                        f"gh pr create --repo {GITHUB_REPOSITORY} "
                        f"--title 'Release pull request for branch {self.release_branch}' "
                        f"--head {self.release_branch} {pr_labels} "
                        "--body 'This PullRequest is a part of Datastore release cycle. "
                        "It is used by CI system only. Do not perform any changes with it.'",
                        dry_run=dry_run,
                        strict=True,
                        verbose=True,
                    )

    def get_change_log_branch(self):
        return f"auto/{self.release_tag}"

    def update_release_info(self, dry_run: bool) -> "ReleaseInfo":
        if self.release_type == "patch":
            if not self.changelog_pr:
                branch = self.get_change_log_branch()
                url = "dry-run" if dry_run else GH.get_pr_url_by_branch(branch=branch, repo=GITHUB_REPOSITORY)
                print(f"ChangeLog PR url [{url}]")
                self.changelog_pr = url
            self.docker = f"docker run --rm clickhouse/clickhouse:{self.version} clickhouse --version"
        else:
            branch = self.get_version_bump_branch()
            url = "dry-run" if dry_run else GH.get_pr_url_by_branch(branch=branch, repo=GITHUB_REPOSITORY)
            print(f"Version bump PR url [{url}]")
            self.version_bump_pr = url

        self.release_url = f"https://github.com/{GITHUB_REPOSITORY}/releases/tag/{self.release_tag}"
        print(f"Release url [{self.release_url}]")
        self.dump()
        return self

    def create_gh_release(self, packages_files: List[str], dry_run: bool) -> None:
        repo = GITHUB_REPOSITORY
        assert repo
        cmd_create = f"gh release create --repo {repo} --title 'Release {self.release_tag}' {self.release_tag}"
        cmds_upload = [
            # `--clobber` makes uploads idempotent: retries succeed even when a
            # previous attempt registered the asset server-side before failing.
            f"gh release upload --clobber {self.release_tag} {file}"
            for file in packages_files
        ]
        if not dry_run:
            # GitHub's release API occasionally returns transient 5xx errors
            # (e.g. 502 Bad Gateway) under load, aborting the whole release.
            # Retry each command with exponential back-off before giving up.
            #
            # `gh release create` is not idempotent: if GitHub creates the
            # release server-side but returns a transient error, the retry hits
            # "already exists" and still fails. Treat that as success by
            # verifying the release exists after any failure.
            if not Shell.check(cmd_create, verbose=True, retries=5):
                # The recovery `gh release view` is invoked precisely when GitHub's
                # API has been flaky. Retry it under the same policy so a single
                # transient 5xx on the verify path doesn't fail the release.
                assert Shell.check(
                    f"gh release view --repo {repo} {self.release_tag}",
                    verbose=True,
                    retries=5,
                ), f"Failed to create release {self.release_tag}"
            # Always upload every asset with `--clobber`. The upload is
            # idempotent (it overwrites an existing asset of the same name), so
            # re-running repairs an asset that a previous attempt uploaded with
            # wrong/partial contents. A name-only "already uploaded" skip would
            # bypass `--clobber` and keep publishing the stale asset.
            for cmd in cmds_upload:
                Shell.check(cmd, strict=True, verbose=True, retries=5)
            self.release_url = (
                f"https://github.com/{GITHUB_REPOSITORY}/releases/tag/{self.release_tag}"
            )
        else:
            print("Dry-run, would run commands:")
            print(f"  * {cmd_create}")
            for cmd in cmds_upload:
                print(f"  * {cmd}")
            self.release_url = "dry-run"
        self.dump()

    def merge_prs(self, dry_run: bool) -> None:
        res = True
        if self.release_type == "patch":
            assert self.changelog_pr
            print("Merging ChangeLog PR")
            changelog_pr_num = 23456 if dry_run else int(self.changelog_pr.rsplit("/", 1)[-1])
            res = Shell.check(
                f"gh pr merge {changelog_pr_num} --repo {GITHUB_REPOSITORY} --merge --auto",
                verbose=True,
                dry_run=dry_run,
                strict=True,
            )
        if self.release_type == "new":
            assert self.version_bump_pr
            print("Merging Version Bump PR")
            version_bump_pr_num = 23456 if dry_run else int(self.version_bump_pr.rsplit("/", 1)[-1])
            res = res and Shell.check(
                f"gh pr merge {version_bump_pr_num} --repo {GITHUB_REPOSITORY} --merge --auto",
                verbose=True,
                dry_run=dry_run,
                strict=True,
            )
        else:
            if not dry_run:
                assert not self.version_bump_pr
        self.prs_merged = res


class RepoTypes:
    RPM = "rpm"
    DEBIAN = "deb"
    TGZ = "tgz"


class PackageDownloader:
    PACKAGES = (
        "clickhouse-client",
        "clickhouse-common-static",
        "clickhouse-common-static-dbg",
        "clickhouse-keeper",
        "clickhouse-keeper-dbg",
        "clickhouse-server",
    )

    PACKAGE_ARCHS = ("amd", "arm")
    MACOS_PACKAGE_TO_BIN_SUFFIX = {
        "amd": "macos",
        "arm": "macos-aarch64",
    }
    LOCAL_DIR = "/tmp/packages"

    @classmethod
    def _get_arch_suffix(cls, package_arch, repo_type):
        if package_arch == "amd":
            return "amd64" if repo_type in (RepoTypes.DEBIAN, RepoTypes.TGZ) else "x86_64"
        if package_arch == "arm":
            return "arm64" if repo_type in (RepoTypes.DEBIAN, RepoTypes.TGZ) else "aarch64"
        assert False, "BUG"

    def __init__(self, release, commit_sha, version):
        assert version.startswith(release), "Invalid release branch or version"
        self.package_names = list(self.PACKAGES)
        self.release = release
        major_version = int(release.split(".")[0])
        minor_version = int(release.split(".")[1])
        self.is_new_ci = (major_version >= 25 and minor_version >= 3) or major_version > 25
        self.s3_release_prefix = f"REFs/{release}" if self.is_new_ci else release
        self.commit_sha = commit_sha
        self.version = version
        self.s3 = S3Helper()
        self.deb_package_files = []
        self.rpm_package_files = []
        self.tgz_package_files = []
        self.macos_package_files = ["clickhouse-macos", "clickhouse-macos-aarch64"]
        self.file_to_job_name = {}
        self.macos_binary_to_job_name = {}

        Shell.check(f"mkdir -p {self.LOCAL_DIR}")

        for package_arch in self.PACKAGE_ARCHS:
            if not self.is_new_ci:
                job_name = "package_release" if package_arch == "amd" else "package_aarch64"
                job_name_darwin = "binary_darwin" if package_arch == "amd" else "binary_darwin_aarch64"
            else:
                job_name = "build_amd_release" if package_arch == "amd" else "build_arm_release"
                job_name_darwin = "build_amd_darwin" if package_arch == "amd" else "build_arm_darwin"

            for package in self.package_names:
                arch = self._get_arch_suffix(package_arch, RepoTypes.DEBIAN)
                deb = f"{package}_{self.version}_{arch}.deb"
                self.deb_package_files.append(deb)
                self.file_to_job_name[deb] = job_name

                arch = self._get_arch_suffix(package_arch, RepoTypes.RPM)
                rpm = f"{package}-{self.version}.{arch}.rpm"
                self.rpm_package_files.append(rpm)
                self.file_to_job_name[rpm] = job_name

                arch = self._get_arch_suffix(package_arch, RepoTypes.TGZ)
                tgz = f"{package}-{self.version}-{arch}.tgz"
                self.tgz_package_files.append(tgz)
                self.file_to_job_name[tgz] = job_name
                tgz_sha = tgz + ".sha512"
                self.tgz_package_files.append(tgz_sha)
                self.file_to_job_name[tgz_sha] = job_name

                dest_bin = f"clickhouse-{self.MACOS_PACKAGE_TO_BIN_SUFFIX[package_arch]}"
                assert dest_bin in self.macos_package_files
                self.macos_binary_to_job_name[dest_bin] = job_name_darwin

    def get_deb_packages_files(self):
        return self.deb_package_files

    def get_rpm_packages_files(self):
        return self.rpm_package_files

    def get_tgz_packages_files(self):
        return self.tgz_package_files

    def get_macos_packages_files(self):
        return self.macos_package_files

    def get_packages_names(self):
        return self.package_names

    def get_all_packages_files(self):
        assert self.local_tgz_packages_ready()
        assert self.local_deb_packages_ready()
        assert self.local_rpm_packages_ready()
        assert self.local_macos_packages_ready()
        res = []
        for pkg in (
            self.deb_package_files
            + self.rpm_package_files
            + self.tgz_package_files
            + self.macos_package_files
        ):
            res.append(self.LOCAL_DIR + "/" + pkg)
        return res

    def run(self):
        # Start from a clean package directory. Package filenames carry the
        # version but not the commit_sha, and a partial/interrupted download
        # leaves a truncated file, so a leftover from an earlier release or a
        # different commit on a reused runner must not be reused — always
        # re-download from S3 for this release commit.
        shutil.rmtree(self.LOCAL_DIR, ignore_errors=True)
        os.makedirs(self.LOCAL_DIR, exist_ok=True)
        for package_file in self.deb_package_files + self.rpm_package_files + self.tgz_package_files:
            local_path = self.LOCAL_DIR + "/" + package_file
            print(f"Downloading: [{package_file}]")
            s3_path = "/".join([
                self.s3_release_prefix,
                self.commit_sha,
                self.file_to_job_name[package_file],
                package_file,
            ])
            self.s3.download_file(
                bucket=S3_BUILDS_BUCKET,
                s3_path=s3_path,
                local_file_path=local_path,
            )

        for macos_binary, job_name in self.macos_binary_to_job_name.items():
            local_path = self.LOCAL_DIR + "/" + macos_binary
            # macOS binaries have fixed, version-less local names (e.g.
            # `clickhouse-macos`). Unlike the deb/rpm/tgz files there is no
            # version in the name to make a stale copy distinguishable, so a
            # leftover file from an earlier release on a reused runner must not
            # be skipped — always re-download to overwrite it.
            print(f"Downloading: [{job_name}] binary to [{macos_binary}]")
            s3_path = "/".join([
                self.s3_release_prefix,
                self.commit_sha,
                job_name,
                "clickhouse",
            ])
            self.s3.download_file(
                bucket=S3_BUILDS_BUCKET,
                s3_path=s3_path,
                local_file_path=local_path,
            )

    def local_deb_packages_ready(self) -> bool:
        return all(Path(self.LOCAL_DIR + "/" + f).is_file() for f in self.deb_package_files)

    def local_rpm_packages_ready(self) -> bool:
        return all(Path(self.LOCAL_DIR + "/" + f).is_file() for f in self.rpm_package_files)

    def local_tgz_packages_ready(self) -> bool:
        return all(Path(self.LOCAL_DIR + "/" + f).is_file() for f in self.tgz_package_files)

    def local_macos_packages_ready(self) -> bool:
        return all(Path(self.LOCAL_DIR + "/" + f).is_file() for f in self.macos_package_files)


@contextmanager
def checkout(ref: str) -> Iterator[None]:
    # Detached-safe: `symbolic-ref` fails when HEAD is already detached (e.g. a
    # nested checkout of a commit SHA), so fall back to the commit itself and
    # restore by re-detaching to it.
    orig_ref = Shell.get_output(
        f"{GIT_PREFIX} symbolic-ref --short HEAD"
    ) or Shell.get_output_or_raise(f"{GIT_PREFIX} rev-parse HEAD")
    rollback_cmd = f"{GIT_PREFIX} checkout {orig_ref}"
    assert orig_ref
    if ref != orig_ref:
        Shell.check(f"{GIT_PREFIX} checkout {ref}", strict=True, verbose=True)
    try:
        yield
    except (Exception, KeyboardInterrupt) as e:
        print(f"ERROR: Exception [{e}]")
        Shell.check(rollback_cmd, verbose=True)
        raise
    Shell.check(rollback_cmd, verbose=True)


@contextmanager
def checkout_new(ref: str) -> Iterator[None]:
    # Detached-safe: `symbolic-ref` fails when HEAD is already detached (e.g. a
    # nested checkout of a commit SHA), so fall back to the commit itself and
    # restore by re-detaching to it.
    orig_ref = Shell.get_output(
        f"{GIT_PREFIX} symbolic-ref --short HEAD"
    ) or Shell.get_output_or_raise(f"{GIT_PREFIX} rev-parse HEAD")
    rollback_cmd = f"{GIT_PREFIX} checkout {orig_ref}"
    assert orig_ref
    # `-B` (create-or-reset) rather than `-b` so a rerun on a preserved
    # self-hosted workspace does not die with "branch already exists" on the
    # leftover branch from a previous attempt; it is reset to the current commit
    # and the caller's force-push / PR-reuse logic proceeds.
    Shell.check(f"{GIT_PREFIX} checkout -B {ref}", strict=True, verbose=True)
    try:
        yield
    except (Exception, KeyboardInterrupt) as e:
        print(f"ERROR: Exception [{e}]")
        Shell.check(rollback_cmd, verbose=True)
        raise
    Shell.check(rollback_cmd, verbose=True)


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        description="Creates release",
    )
    parser.add_argument(
        "--prepare-release-info",
        action="store_true",
        help="Initial step to prepare info like release branch, release tag, etc.",
    )
    parser.add_argument(
        "--push-release-tag",
        action="store_true",
        help="Creates and pushes git tag",
    )
    parser.add_argument(
        "--push-new-release-branch",
        action="store_true",
        help="Creates and pushes new release branch and corresponding service gh tags for backports",
    )
    parser.add_argument(
        "--create-bump-version-pr",
        action="store_true",
        help="Updates version, contributors list and creates PR",
    )
    parser.add_argument(
        "--download-packages",
        action="store_true",
        help="Downloads all required packages from s3",
    )
    parser.add_argument(
        "--create-gh-release",
        action="store_true",
        help="Create GH Release object and attach all packages",
    )
    parser.add_argument(
        "--merge-prs",
        action="store_true",
        help="Merge PRs with version, changelog updates",
    )
    parser.add_argument(
        "--post-status",
        action="store_true",
        help="Post release status (prints summary; Slack integration removed)",
    )
    parser.add_argument(
        "--ref",
        type=str,
        help="the commit hash or branch",
    )
    parser.add_argument(
        "--release-type",
        choices=("new", "patch"),
        help="a release type to bump the major.minor.patch version part",
    )
    parser.add_argument(
        "--dry-run",
        action="store_true",
        help="do not make any actual changes in the repo, just show what will be done",
    )
    return parser.parse_args()


if __name__ == "__main__":
    args = parse_args()

    _ssh_agent = None
    _key_pub = None
    if os.getenv("ROBOT_CLICKHOUSE_SSH_KEY", ""):
        _key = os.getenv("ROBOT_CLICKHOUSE_SSH_KEY")
        _ssh_agent = SSHAgent()
        _key_pub = _ssh_agent.add(_key)
        _ssh_agent.print_keys()

    if args.prepare_release_info:
        with ReleaseContextManager(release_progress=ReleaseProgress.STARTED) as release_info:
            assert (
                args.ref and args.release_type
            ), "--ref and --release-type must be provided with --prepare-release-info"
            release_info.prepare(
                commit_ref=args.ref,
                release_type=args.release_type,
                dry_run=args.dry_run,
            )

    if args.download_packages:
        with ReleaseContextManager(
            release_progress=ReleaseProgress.DOWNLOAD_PACKAGES
        ) as release_info:
            p = PackageDownloader(
                release=release_info.release_branch,
                commit_sha=release_info.commit_sha,
                version=release_info.version,
            )
            p.run()

    if args.push_release_tag:
        with ReleaseContextManager(
            release_progress=ReleaseProgress.PUSH_RELEASE_TAG
        ) as release_info:
            release_info.push_release_tag(dry_run=args.dry_run)

    if args.push_new_release_branch:
        with ReleaseContextManager(
            release_progress=ReleaseProgress.PUSH_NEW_RELEASE_BRANCH
        ) as release_info:
            release_info.push_new_release_branch(dry_run=args.dry_run)

    if args.create_bump_version_pr:
        with ReleaseContextManager(
            release_progress=ReleaseProgress.BUMP_VERSION
        ) as release_info:
            release_info.update_version_and_contributors_list(dry_run=args.dry_run)

    if args.create_gh_release:
        with ReleaseContextManager(
            release_progress=ReleaseProgress.CREATE_GH_RELEASE
        ) as release_info:
            p = PackageDownloader(
                release=release_info.release_branch,
                commit_sha=release_info.commit_sha,
                version=release_info.version,
            )
            release_info.create_gh_release(
                packages_files=p.get_all_packages_files(), dry_run=args.dry_run
            )

    if args.post_status:
        release_info = ReleaseInfo.from_file()
        if release_info.is_new_release_branch():
            title = "New release branch"
        else:
            title = "New release"
        # Print the release-info summary for the job log / Slack context. Pass or
        # fail is conveyed by the workflow job result, not re-derived here.
        print(f"{title}: {release_info.release_tag}")
        print(json.dumps(dataclasses.asdict(release_info), indent=2))

    if args.merge_prs:
        with ReleaseContextManager(
            release_progress=ReleaseProgress.MERGE_CREATED_PRS
        ) as release_info:
            release_info.update_release_info(dry_run=args.dry_run)
            release_info.merge_prs(dry_run=args.dry_run)

    if _ssh_agent and _key_pub:
        _ssh_agent.remove(_key_pub)