mirror of
https://github.com/hanzoai/visor.git
synced 2026-09-24 17:04:14 +00:00
Ryan El KochtaandgVisor bot
0d82c7283d
Low storage space is a primary reason that `docker kill` might stall, so this will allow us to diagnose clear_docker_containers failures more clearly. PiperOrigin-RevId: 972891010
47 lines
1.6 KiB
Bash
47 lines
1.6 KiB
Bash
#!/bin/bash
|
|
|
|
# Copyright 2025 The gVisor Authors.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
kill_agent() {
|
|
echo "$1; node is bad." >&2
|
|
echo "Automated monitoring should recycle this node soon." >&2
|
|
echo "If this made your build pipeline fail, sorry!" >&2
|
|
echo "Try your luck again." >&2
|
|
sleep 10
|
|
killall buildkite-agent
|
|
exit 1
|
|
}
|
|
|
|
clear_docker_containers() { # $1: "kill_agent" to kill the agent on failure
|
|
local on_fail="${1:-}" id rc
|
|
# Kill any running containers (clear state), except for "bootstrap".
|
|
for id_and_name in $(docker ps --format='{{.ID}}/{{.Names}}'); do
|
|
if [[ "$(echo "$id_and_name" | cut -d'/' -f2-)" == 'bootstrap' ]]; then
|
|
continue
|
|
fi
|
|
id="$(echo "$id_and_name" | cut -d'/' -f1)"
|
|
rc=0
|
|
timeout --kill-after=10s --preserve-status 30s \
|
|
docker container kill "$id" || rc=$?
|
|
if [[ "${rc}" -ne 0 ]]; then
|
|
if [[ "${on_fail}" == 'kill_agent' ]]; then
|
|
kill_agent "Docker container kill failed for ${id} (exit ${rc})"
|
|
else
|
|
echo "WARN: docker container kill failed for ${id} (exit ${rc})" >&2
|
|
fi
|
|
fi
|
|
done
|
|
}
|