mirror of
https://github.com/hanzoai/datastore.git
synced 2026-09-26 13:55:07 +00:00
Max KainovandClaude Sonnet 4.6
12d765bdf1
Replaces the sequential `prefetch_images` loop in `ci/jobs/integration_test_job.py` with a new shell script `ci/jobs/scripts/prefetch-integration-test-images` that: - Pulls all images in parallel (background subshells + `wait`). - Treats `no matching manifest` errors as a non-fatal skip rather than a failure, so amd64-only images (e.g. `clickhouse/kerberos-kdc`) do not abort the job on arm64 runners. Also fixes `get_images_from_compose_files` to strip inline YAML comments (e.g. `coredns/coredns:1.9.3 # :latest broke this test`) before passing the value to `docker pull`. CI report: https://s3.amazonaws.com/clickhouse-test-reports/json.html?PR=100039&sha=253900cdb6028e70c4f55b6792a0f251811ad9cd&name_0=PR&name_1=Integration%20tests%20%28arm_binary%2C%20distributed%20plan%2C%201%2F4%29 Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
99 lines
2.7 KiB
Bash
Executable File
99 lines
2.7 KiB
Bash
Executable File
#!/bin/bash
|
|
# Pre-fetches Docker images in parallel before integration tests start.
|
|
#
|
|
# Images that are unavailable for the current architecture (e.g. amd64-only
|
|
# images on arm64 runners) are silently skipped rather than failing the job.
|
|
#
|
|
# Usage:
|
|
# ci/prefetch-integration-test-images IMAGE [IMAGE ...]
|
|
#
|
|
# Environment variables:
|
|
# PULL_RETRIES Number of pull attempts per image (default: 3)
|
|
# PULL_TIMEOUT Per-attempt timeout in seconds (default: 300)
|
|
|
|
set -uo pipefail
|
|
|
|
PULL_RETRIES="${PULL_RETRIES:-3}"
|
|
PULL_TIMEOUT="${PULL_TIMEOUT:-300}"
|
|
|
|
images=("$@")
|
|
if [[ ${#images[@]} -eq 0 ]]; then
|
|
echo "No images to pre-fetch."
|
|
exit 0
|
|
fi
|
|
|
|
echo "Pre-fetching ${#images[@]} Docker image(s) in parallel:"
|
|
for img in "${images[@]}"; do
|
|
echo " $img"
|
|
done
|
|
|
|
work_dir="$(mktemp -d)"
|
|
trap 'rm -rf "$work_dir"' EXIT
|
|
|
|
pull_one()
|
|
{
|
|
local image="$1"
|
|
local fail_file="$2"
|
|
local attempt out start elapsed
|
|
|
|
for ((attempt = 1; attempt <= PULL_RETRIES; attempt++)); do
|
|
echo "Pulling $image (attempt $attempt/$PULL_RETRIES) ..."
|
|
start=$SECONDS
|
|
if out=$(timeout "$PULL_TIMEOUT" docker pull "$image" 2>&1); then
|
|
elapsed=$((SECONDS - start))
|
|
echo "Pulled $image in ${elapsed}s"
|
|
return 0
|
|
fi
|
|
elapsed=$((SECONDS - start))
|
|
# Arch-specific image — not a real failure on this runner.
|
|
if echo "$out" | grep -q "no matching manifest"; then
|
|
echo "SKIP $image: no manifest for this architecture (${elapsed}s)"
|
|
return 0
|
|
fi
|
|
echo "Pull of $image failed on attempt $attempt (${elapsed}s elapsed):"
|
|
echo "$out"
|
|
if ((attempt < PULL_RETRIES)); then
|
|
sleep 5
|
|
fi
|
|
done
|
|
|
|
echo "FAILED to pull $image after $PULL_RETRIES attempt(s)"
|
|
echo "$image" > "$fail_file"
|
|
return 1
|
|
}
|
|
|
|
# Launch all pulls in parallel; each writes its image name to a unique file on failure.
|
|
pids=()
|
|
for image in "${images[@]}"; do
|
|
# Sanitize image name to a safe filename.
|
|
safe="${image//[^a-zA-Z0-9._-]/_}"
|
|
pull_one "$image" "$work_dir/fail_${safe}" &
|
|
pids+=($!)
|
|
done
|
|
|
|
# Collect exit statuses.
|
|
any_fail=0
|
|
for pid in "${pids[@]}"; do
|
|
wait "$pid" || any_fail=1
|
|
done
|
|
|
|
# Report failures.
|
|
failed=()
|
|
for f in "$work_dir"/fail_*; do
|
|
[[ -s "$f" ]] && failed+=("$(cat "$f")")
|
|
done
|
|
|
|
if [[ ${#failed[@]} -gt 0 ]]; then
|
|
echo "ERROR: Failed to pull the following image(s) after $PULL_RETRIES attempt(s): ${failed[*]}"
|
|
exit 1
|
|
fi
|
|
|
|
if [[ $any_fail -ne 0 ]]; then
|
|
# A pull_one returned non-zero without writing a fail file — shouldn't happen,
|
|
# but guard against it anyway.
|
|
echo "ERROR: One or more image pulls failed unexpectedly."
|
|
exit 1
|
|
fi
|
|
|
|
echo "All images pre-fetched successfully."
|