mirror of
https://github.com/hanzoai/visor.git
synced 2026-09-24 17:04:14 +00:00
Add building the Python SandboxExec wheel to our build pipeline
Successful BuildKite run: https://buildkite.com/gvisor/pipeline/builds/46443/list?sid=01a01bfa-83bd-4ea4-b075-176caf09f401&tab=output PiperOrigin-RevId: 974634114
This commit is contained in:
Caroline Zhu
@@ -52,6 +52,7 @@ steps:
|
||||
commands:
|
||||
- "make BAZEL_OPTIONS='--config=x86_64 --compilation_mode=opt' artifacts/x86_64"
|
||||
- "make BAZEL_OPTIONS='--config=aarch64 --compilation_mode=opt' artifacts/aarch64"
|
||||
- make artifacts-python
|
||||
- make release RELEASE_NIGHTLY=$$RELEASE_NIGHTLY
|
||||
- cd repo && gcloud storage cp --recursive . gs://gvisor/releases/
|
||||
agents:
|
||||
|
||||
@@ -868,6 +868,11 @@ $(RELEASE_ARTIFACTS)/%:
|
||||
@$(call copy,//debian:gvisor-release-tar-bz2,$@)
|
||||
@$(call copy,//debian:gvisor-release-tar-zstd,$@)
|
||||
|
||||
artifacts-python: ensure-bazel-server ## Builds Python SandboxExec wheels into $(RELEASE_ARTIFACTS)/python.
|
||||
@mkdir -p $(RELEASE_ARTIFACTS)/python
|
||||
@$(call wrapper,tools/make_python_release.sh build $(RELEASE_ARTIFACTS)/python "$(RELEASE_NAME)")
|
||||
.PHONY: artifacts-python
|
||||
|
||||
release: $(RELEASE_KEY) $(RELEASE_ARTIFACTS)/$(ARCH)
|
||||
@mkdir -p $(RELEASE_ROOT)
|
||||
@NIGHTLY=$(RELEASE_NIGHTLY) tools/make_release.sh $(RELEASE_KEY) $(RELEASE_ROOT) $$(find $(RELEASE_ARTIFACTS) -type f)
|
||||
|
||||
@@ -9,6 +9,7 @@ ENV DEBIAN_FRONTEND="noninteractive"
|
||||
#-------------------------------------------------------------------------------
|
||||
RUN apt-get update && apt-get install -y curl gnupg2 git \
|
||||
python-is-python3 python3 python3-setuptools python3-pip \
|
||||
python3-build python3-wheel python3-venv \
|
||||
build-essential crossbuild-essential-arm64 qemu-user-static \
|
||||
openjdk-11-jdk-headless zip unzip \
|
||||
apt-transport-https ca-certificates gnupg-agent \
|
||||
|
||||
Executable
+246
@@ -0,0 +1,246 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Copyright 2026 The gVisor Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -xeuo pipefail
|
||||
|
||||
usage() {
|
||||
echo "usage: $0 <command> [args...]" >&2
|
||||
echo "Commands:" >&2
|
||||
echo " build <dest-dir> [release-name]" >&2
|
||||
echo " upload-wheel <target-dir>" >&2
|
||||
echo " all <dest-dir> <target-dir> [release-name]" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# get_next_ar_version queries Artifact Registry for today's gvisor versions
|
||||
# and returns YYYY.MM.DD.<max_rev + 1>, or YYYY.MM.DD.0 if none exist.
|
||||
get_next_ar_version() {
|
||||
local -r today="$(date +%Y.%m.%d)"
|
||||
local versions=""
|
||||
|
||||
echo "=== [Version Discovery] Checking Artifact Registry for today's versions (${today}) ===" >&2
|
||||
|
||||
# Query AR via gcloud CLI
|
||||
if command -v gcloud >/dev/null 2>&1; then
|
||||
echo "Querying versions via: gcloud artifacts versions list --package=gvisor --repository=gvisor--pypi --location=us --project=oss-exit-gate-prod" >&2
|
||||
versions=$(gcloud artifacts versions list \
|
||||
--package=gvisor \
|
||||
--repository=gvisor--pypi \
|
||||
--location=us \
|
||||
--project=oss-exit-gate-prod \
|
||||
--format="value(version)" 2>/dev/null || true)
|
||||
fi
|
||||
|
||||
if [[ -n "${versions}" ]]; then
|
||||
echo "Found existing versions in Artifact Registry:" >&2
|
||||
echo "${versions}" >&2
|
||||
else
|
||||
echo "No existing versions found in Artifact Registry for today (${today})." >&2
|
||||
fi
|
||||
|
||||
# Find the highest revision for today
|
||||
local max_rev=""
|
||||
if [[ -n "${versions}" ]]; then
|
||||
max_rev=$(echo "${versions}" | grep -E "^${today}\.[0-9]+$" | sed "s/^${today}\.//" | sort -n | tail -n 1)
|
||||
fi
|
||||
|
||||
if [[ -n "${max_rev}" ]]; then
|
||||
local -r next_rev=$((max_rev + 1))
|
||||
echo "Highest existing revision: ${max_rev}. Next revision: ${next_rev}." >&2
|
||||
echo "${today}.${next_rev}"
|
||||
else
|
||||
echo "Starting at revision 0 for today: ${today}.0" >&2
|
||||
echo "${today}.0"
|
||||
fi
|
||||
}
|
||||
|
||||
# update_version converts release-YYYYMMDD.x to PEP 440 CalVer format YYYY.MM.DD.x
|
||||
# or resolves auto version from AR, and updates pyproject.toml in-place.
|
||||
update_version() {
|
||||
local -r release_name="${1:-}"
|
||||
local wheel_version=""
|
||||
|
||||
echo "=== [Version Injection] Updating pyproject.toml version ==="
|
||||
if [[ "${release_name}" == "auto" ]] || [[ -z "${release_name}" ]]; then
|
||||
echo "Resolving next dynamic version from Artifact Registry..."
|
||||
wheel_version=$(get_next_ar_version)
|
||||
elif [[ "${release_name}" =~ ^([0-9]{4})\.([0-9]{2})\.([0-9]{2})\.(.*)$ ]]; then
|
||||
# Already in YYYY.MM.DD.x format
|
||||
wheel_version="${release_name}"
|
||||
else
|
||||
# Convert release-YYYYMMDD.x or YYYYMMDD.x to YYYY.MM.DD.x
|
||||
wheel_version=$(echo "${release_name#release-}" | sed -E 's/^([0-9]{4})([0-9]{2})([0-9]{2})\.(.*)$/\1.\2.\3.\4/')
|
||||
fi
|
||||
|
||||
if [[ -n "${wheel_version}" ]]; then
|
||||
echo "Updating pyproject.toml version to: ${wheel_version}"
|
||||
sed -i "s/^version = \".*\"/version = \"${wheel_version}\"/" pyproject.toml
|
||||
echo "pyproject.toml version is now:"
|
||||
grep "^version = " pyproject.toml || true
|
||||
fi
|
||||
}
|
||||
|
||||
# build_wheel_sdist builds Python SandboxExec wheel and sdist archives and copies them to dest-dir.
|
||||
build_wheel_sdist() {
|
||||
local -r dest_dir="$1"
|
||||
local -r release_name="${2:-}"
|
||||
|
||||
echo "=== [Build] Building Python SandboxExec Wheel & SDist ==="
|
||||
mkdir -p "${dest_dir}"
|
||||
local -r abs_dest_dir="$(cd "${dest_dir}" && pwd)"
|
||||
cd "$(dirname "$0")/../sandboxexec/sandbox/python"
|
||||
echo "Working directory: $(pwd)"
|
||||
echo "Target destination directory: ${abs_dest_dir}"
|
||||
|
||||
update_version "${release_name}"
|
||||
|
||||
# Build wheel and sdist (with fallback to --no-isolation or uv build)
|
||||
echo "Building distribution packages..."
|
||||
if python3 -m build 2>/dev/null; then
|
||||
echo "Successfully built wheel and sdist with python3 -m build."
|
||||
elif python3 -m build --no-isolation 2>/dev/null; then
|
||||
echo "Successfully built wheel and sdist with python3 -m build --no-isolation."
|
||||
elif command -v uv >/dev/null 2>&1; then
|
||||
echo "python3 -m build not available, building with uv build..."
|
||||
uv build
|
||||
else
|
||||
echo "ERROR: Neither python3 -m build nor uv build found to build Python artifacts." >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "Generated artifacts in dist/:"
|
||||
ls -lh dist/
|
||||
|
||||
echo "Copying artifacts to ${abs_dest_dir}..."
|
||||
cp -f dist/*.whl "${abs_dest_dir}/"
|
||||
cp -f dist/*.tar.gz "${abs_dest_dir}/"
|
||||
|
||||
echo "Staged Python artifacts in ${abs_dest_dir}:"
|
||||
ls -lh "${abs_dest_dir}"
|
||||
echo "=== [Build] Python build complete! ==="
|
||||
}
|
||||
|
||||
# ensure_twine_prerequisites checks if twine is available on the system.
|
||||
# If missing, it installs python3-pip, keyring, keyrings.google-artifactregistry-auth,
|
||||
# and twine>=5.0.0.
|
||||
ensure_twine_prerequisites() {
|
||||
export PATH="${HOME}/.local/bin:/usr/local/bin:${PATH}"
|
||||
|
||||
if command -v twine >/dev/null 2>&1 || python3 -c "import twine" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "=== [Bootstrap] twine not found on PATH. Installing publishing prerequisites... ===" >&2
|
||||
if ! command -v pip3 >/dev/null 2>&1; then
|
||||
echo "pip3 not found. Attempting to install python3-pip..." >&2
|
||||
if command -v sudo >/dev/null 2>&1 && command -v apt-get >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y python3-pip
|
||||
elif command -v apt-get >/dev/null 2>&1; then
|
||||
apt-get update && apt-get install -y python3-pip
|
||||
fi
|
||||
fi
|
||||
|
||||
if command -v pip3 >/dev/null 2>&1; then
|
||||
echo "Installing keyring, keyrings.google-artifactregistry-auth, and twine..." >&2
|
||||
pip3 install --quiet --no-cache-dir keyring keyrings.google-artifactregistry-auth "twine>=5.0.0" 2>/dev/null || \
|
||||
sudo pip3 install --quiet --no-cache-dir keyring keyrings.google-artifactregistry-auth "twine>=5.0.0" 2>/dev/null || \
|
||||
pip3 install --quiet --user --no-cache-dir keyring keyrings.google-artifactregistry-auth "twine>=5.0.0" 2>/dev/null || true
|
||||
elif command -v pip >/dev/null 2>&1; then
|
||||
pip install --quiet --no-cache-dir keyring keyrings.google-artifactregistry-auth "twine>=5.0.0" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
export PATH="${HOME}/.local/bin:/usr/local/bin:${PATH}"
|
||||
}
|
||||
|
||||
# upload_to_ar uploads staged Python wheel and sdist files to OSS Exit Gate Artifact Registry.
|
||||
upload_to_ar() {
|
||||
local -r target_dir="$1"
|
||||
|
||||
ensure_twine_prerequisites
|
||||
|
||||
echo "=== [Upload] Uploading Python artifacts to OSS Exit Gate Artifact Registry ==="
|
||||
if [[ ! -d "${target_dir}" ]]; then
|
||||
echo "ERROR: Target directory ${target_dir} does not exist." >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
local -r abs_target_dir="$(cd "${target_dir}" && pwd)"
|
||||
echo "Inspecting artifacts in ${abs_target_dir}..."
|
||||
ls -lh "${abs_target_dir}"
|
||||
|
||||
local whl_count tar_count
|
||||
whl_count=$(find "${abs_target_dir}" -maxdepth 1 -name '*.whl' 2>/dev/null | wc -l)
|
||||
tar_count=$(find "${abs_target_dir}" -maxdepth 1 -name '*.tar.gz' 2>/dev/null | wc -l)
|
||||
if [[ "${whl_count}" -eq 0 ]] && [[ "${tar_count}" -eq 0 ]]; then
|
||||
echo "ERROR: No .whl or .tar.gz files found in ${abs_target_dir}." >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "Found ${whl_count} wheel(s) and ${tar_count} source distribution(s) to upload."
|
||||
echo "Publishing artifacts to https://us-python.pkg.dev/oss-exit-gate-prod/gvisor--pypi via twine..."
|
||||
if command -v twine >/dev/null 2>&1; then
|
||||
echo "Using twine upload..."
|
||||
twine upload \
|
||||
--repository-url https://us-python.pkg.dev/oss-exit-gate-prod/gvisor--pypi \
|
||||
"${abs_target_dir}"/*.whl "${abs_target_dir}"/*.tar.gz
|
||||
elif python3 -c "import twine" >/dev/null 2>&1; then
|
||||
echo "Using python3 -m twine upload..."
|
||||
python3 -m twine upload \
|
||||
--repository-url https://us-python.pkg.dev/oss-exit-gate-prod/gvisor--pypi \
|
||||
"${abs_target_dir}"/*.whl "${abs_target_dir}"/*.tar.gz
|
||||
else
|
||||
echo "ERROR: twine not found to publish Python artifacts." >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "=== [Upload] Successfully uploaded all artifacts to Artifact Registry! ==="
|
||||
}
|
||||
|
||||
main() {
|
||||
if [[ "$#" -lt 1 ]]; then
|
||||
usage
|
||||
fi
|
||||
|
||||
local -r cmd="$1"
|
||||
shift
|
||||
|
||||
case "${cmd}" in
|
||||
build)
|
||||
if [[ "$#" -lt 1 ]]; then
|
||||
usage
|
||||
fi
|
||||
build_wheel_sdist "$1" "${2:-}"
|
||||
;;
|
||||
upload-wheel)
|
||||
if [[ "$#" -lt 1 ]]; then
|
||||
usage
|
||||
fi
|
||||
upload_to_ar "$1"
|
||||
;;
|
||||
all)
|
||||
if [[ "$#" -lt 2 ]]; then
|
||||
usage
|
||||
fi
|
||||
build_wheel_sdist "$1" "${3:-}"
|
||||
upload_to_ar "$2"
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
main "$@"
|
||||
+15
-4
@@ -46,11 +46,19 @@ install_raw() {
|
||||
# Copy the raw file & generate a sha512sum, sorted by architecture.
|
||||
# For tarballs, determine arch from the `runsc` within the tarball.
|
||||
case "${binary}" in
|
||||
*.tar.bz2) file_info=$(tar -xjOf "${binary}" runsc | file -) ;;
|
||||
*.tar.zstd) file_info=$(tar --zstd -xOf "${binary}" runsc | file -) ;;
|
||||
*) file_info=$(file "${binary}") ;;
|
||||
*.tar.bz2)
|
||||
arch=$(tar -xjOf "${binary}" runsc | file - | cut -d',' -f2 | awk '{print $NF}' | tr '-' '_')
|
||||
;;
|
||||
*.tar.zstd)
|
||||
arch=$(tar --zstd -xOf "${binary}" runsc | file - | cut -d',' -f2 | awk '{print $NF}' | tr '-' '_')
|
||||
;;
|
||||
*.whl|*.tar.gz)
|
||||
arch="python"
|
||||
;;
|
||||
*)
|
||||
arch=$(file "${binary}" | cut -d',' -f2 | awk '{print $NF}' | tr '-' '_')
|
||||
;;
|
||||
esac
|
||||
arch=$(echo "${file_info}" | cut -d',' -f2 | awk '{print $NF}' | tr '-' '_')
|
||||
name=$(basename "${binary}")
|
||||
mkdir -p "${root}/$1/${arch}"
|
||||
cp -f "${binary}" "${root}/$1/${arch}"
|
||||
@@ -63,6 +71,8 @@ install_apt() {
|
||||
tools/make_apt.sh "${private_key}" "$1" "${root}" "${pkgs[@]}"
|
||||
}
|
||||
|
||||
|
||||
|
||||
# If nightly, install only nightly artifacts.
|
||||
if [[ "${NIGHTLY:-false}" == "true" ]]; then
|
||||
# Install the nightly release.
|
||||
@@ -92,6 +102,7 @@ else
|
||||
# Install the "point release".
|
||||
# https://gvisor.dev/docs/user_guide/install/#point-release
|
||||
install_raw "release/${name}"
|
||||
tools/make_python_release.sh upload-wheel "${root}/release/${name}/python"
|
||||
# Install the latest release.
|
||||
# https://gvisor.dev/docs/user_guide/install/#latest-release
|
||||
install_raw "release/latest"
|
||||
|
||||
Reference in New Issue
Block a user