hanzoai/visor
2
0
Fork 0
mirror of https://github.com/hanzoai/visor.git synced 2026-09-24 17:04:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor the Makefile to avoid recursive Make.

Browse Source 
Recursive make is difficult to follow and debug. Drop this by using
internal functions, which, while difficult, are easier than trying to
following recursive invokations.

Further simplify the Makefile by collapsing the image bits and removing
the tools/vm directory, which is effectively unused.

Fixes #4952

PiperOrigin-RevId: 346569133
This commit is contained in:
Adin Scannell
2020-12-09 15:53:23 -08:00
committed by Shentubot
parent 9835e2f598
commit 0bcbf188c6
28 changed files with 496 additions and 1403 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.buildkite/pipeline.yaml
+1 -3
View File
@@ -17,9 +17,7 @@ steps:
- <<: *common
label: ":golang: Go branch"
commands:
- rm -rf bazel-bin/gopath
- make build TARGETS="//:gopath"
- tools/go_branch.sh
- make go
- git checkout go && git clean -f
- go build ./...
.gitignore
+4 -1
View File
@@ -1,4 +1,7 @@
# Generated bazel symlinks.
/bazel-*
# Generated build event file.
/.build_events.json
/.build_events.json
# Generated repository.
/repo
/repo.key
Makefile
+185 -221
View File
@@ -14,27 +14,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Helpful pretty-printer.
ifeq (0,$(MAKELEVEL))
OPENLAST := || (rc=$$?; echo '^^^ +++' >&2; exit $$rc)
else
OPENLAST :=
endif
CMDLINE := $(shell cut -d '' -f2- /proc/$$PPID/cmdline | sed 's|\x00| |g')
submake = echo '--- make $1' >&2 && \
$(MAKE) -s $1 && \
echo '--- make $(CMDLINE) (resume)' >&2 \
$(OPENLAST)
# Described below.
OPTIONS :=
STARTUP_OPTIONS :=
TARGETS := //runsc
ARGS :=
default: runsc
.PHONY: default
# Header for debugging (used by other macros).
header = echo --- $(1) >&2
# Make hacks.
EMPTY :=
SPACE := $(EMPTY) $(EMPTY)
## usage: make <target>
## or
## make <build|test|copy|run|sudo> STARTUP_OPTIONS="..." OPTIONS="..." TARGETS="..." ARGS="..."
@@ -46,7 +35,6 @@ default: runsc
## requirements.
##
## There are common arguments that may be passed to targets. These are:
## STARTUP_OPTIONS - Bazel startup options.
## OPTIONS - Build or test options.
## TARGETS - The bazel targets.
## ARGS - Arguments for run or sudo.
@@ -57,7 +45,7 @@ default: runsc
## make build OPTIONS="" TARGETS="//runsc"'
##
help: ## Shows all targets and help from the Makefile (this message).
@grep --no-filename -E '^([a-z.A-Z_-]+:.*?|)##' $(MAKEFILE_LIST) | \
@grep --no-filename -E '^([a-z.A-Z_%-]+:.*?|)##' $(MAKEFILE_LIST) | \
awk 'BEGIN {FS = "(:.*?|)## ?"}; { \
if (length($$1) > 0) { \
printf " \033[36m%-20s\033[0m %s\n", $$1, $$2; \
@@ -65,17 +53,34 @@ help: ## Shows all targets and help from the Makefile (this message).
printf "%s\n", $$2; \
} \
}'
build: ## Builds the given $(TARGETS) with the given $(OPTIONS). E.g. make build TARGETS=runsc
test: ## Tests the given $(TARGETS) with the given $(OPTIONS). E.g. make test TARGETS=pkg/buffer:buffer_test
copy: ## Copies the given $(TARGETS) to the given $(DESTINATION). E.g. make copy TARGETS=runsc DESTINATION=/tmp
run: ## Runs the given $(TARGETS), built with $(OPTIONS), using $(ARGS). E.g. make run TARGETS=runsc ARGS=-version
sudo: ## Runs the given $(TARGETS) as per run, but using "sudo -E". E.g. make sudo TARGETS=test/root:root_test ARGS=-test.v
.PHONY: help build test copy run sudo
@$(call build,$(OPTIONS) $(TARGETS))
.PHONY: build
test: ## Tests the given $(TARGETS) with the given $(OPTIONS). E.g. make test TARGETS=pkg/buffer:buffer_test
@$(call build,$(OPTIONS) $(TARGETS))
.PHONY: test
copy: ## Copies the given $(TARGETS) to the given $(DESTINATION). E.g. make copy TARGETS=runsc DESTINATION=/tmp
@$(call copy,$(TARGETS),$(DESTINATION))
.PHONY: copy
run: ## Runs the given $(TARGETS), built with $(OPTIONS), using $(ARGS). E.g. make run TARGETS=runsc ARGS=-version
@$(call build,$(TARGETS) $(ARGS))
.PHONY: run
sudo: ## Runs the given $(TARGETS) as per run, but using "sudo -E". E.g. make sudo TARGETS=test/root:root_test ARGS=-test.v
@$(call sudo,$(TARGETS) $(ARGS))
.PHONY: sudo
# Load image helpers.
include tools/images.mk
# Load all bazel wrappers.
#
# This file should define the basic "build", "test", "run" and "sudo" rules, in
# addition to the $(BRANCH_NAME) variable.
# addition to the $(BRANCH_NAME) and $(BUILD_ROOTS) variables.
ifneq (,$(wildcard tools/google.mk))
include tools/google.mk
else
@@ -83,32 +88,71 @@ include tools/bazel.mk
endif
##
## Docker image targets.
## Development helpers and tooling.
##
## Images used by the tests must also be built and available locally.
## The canonical test targets defined below will automatically load
## relevant images. These can be loaded or built manually via these
## targets.
## These targets faciliate local development by automatically
## installing and configuring a runtime. Several variables may
## be used here to tweak the installation:
## RUNTIME - The name of the installed runtime (default: branch).
## RUNTIME_DIR - Where the runtime will be installed (default: temporary directory with the $RUNTIME).
## RUNTIME_BIN - The runtime binary (default: $RUNTIME_DIR/runsc).
## RUNTIME_LOG_DIR - The logs directory (default: $RUNTIME_DIR/logs).
## RUNTIME_LOGS - The log pattern (default: $RUNTIME_LOG_DIR/runsc.log.%TEST%.%TIMESTAMP%.%COMMAND%).
##
## (*) Note that you may provide an ARCH parameter in order to build
## and load images from an alternate archiecture (using qemu). When
## bazel is run as a server, this has the effect of running an full
## cross-architecture chain, and can produce cross-compiled binaries.
##
define images
$(1)-%: ## Image tool: $(1) a given image (also may use 'all-images').
@$(call submake,-C images $$@)
endef
rebuild-...: ## Rebuild the given image. Also may use 'rebuild-all-images'.
$(eval $(call images,rebuild))
push-...: ## Push the given image. Also may use 'push-all-images'.
$(eval $(call images,push))
pull-...: ## Pull the given image. Also may use 'pull-all-images'.
$(eval $(call images,pull))
load-...: ## Load (pull or rebuild) the given image. Also may use 'load-all-images'.
$(eval $(call images,load))
list-images: ## List all available images.
@$(call submake, -C images $$@)
ifeq (,$(BRANCH_NAME))
RUNTIME := runsc
RUNTIME_DIR := $(shell dirname $(shell mktemp -u))/$(RUNTIME)
else
RUNTIME := $(BRANCH_NAME)
RUNTIME_DIR := $(shell dirname $(shell mktemp -u))/$(RUNTIME)
endif
RUNTIME_BIN := $(RUNTIME_DIR)/runsc
RUNTIME_LOG_DIR := $(RUNTIME_DIR)/logs
RUNTIME_LOGS := $(RUNTIME_LOG_DIR)/runsc.log.%TEST%.%TIMESTAMP%.%COMMAND%
$(RUNTIME_BIN): # See below.
@mkdir -p "$(RUNTIME_DIR)"
@$(call copy,//runsc,$(RUNTIME_BIN))
.PHONY: $(RUNTIME_BIN) # Real file, but force rebuild.
# Configure helpers for below.
configure_noreload = \
$(call header,CONFIGURE $(1) → $(RUNTIME_BIN) $(2)); \
sudo $(RUNTIME_BIN) install --experimental=true --runtime="$(1)" -- --debug-log "$(RUNTIME_LOGS)" $(2) && \
sudo rm -rf "$(RUNTIME_LOG_DIR)" && mkdir -p "$(RUNTIME_LOG_DIR)"
reload_docker = \
sudo systemctl reload docker && \
if test -f /etc/docker/daemon.json; then \
sudo chmod 0755 /etc/docker && \
sudo chmod 0644 /etc/docker/daemon.json; \
fi
configure = $(call configure_noreload,$(1),$(2)) && $(reload_docker)
# Helpers for above. Requires $(RUNTIME_BIN) dependency.
install_runtime = $(call configure,$(RUNTIME),$(1) --TESTONLY-test-name-env=RUNSC_TEST_NAME)
test_runtime = $(call test,--test_arg=--runtime=$(RUNTIME) $(PARTITIONS) $(1))
dev: $(RUNTIME_BIN) ## Installs a set of local runtimes. Requires sudo.
@$(call configure_noreload,$(RUNTIME),--net-raw)
@$(call configure_noreload,$(RUNTIME)-d,--net-raw --debug --strace --log-packets)
@$(call configure_noreload,$(RUNTIME)-p,--net-raw --profile)
@$(call configure_noreload,$(RUNTIME)-vfs2-d,--net-raw --debug --strace --log-packets --vfs2)
@$(call reload_docker)
.PHONY: dev
nogo: ## Surfaces all nogo findings.
@$(call build,--build_tag_filters nogo //...)
@$(call run,//tools/github $(foreach dir,$(BUILD_ROOTS),-path=$(CURDIR)/$(dir)) -dry-run nogo)
.PHONY: nogo
go: ## Builds the Go branch.
@$(call clean)
@$(call build,//:gopath)
@tools/go_branch.sh
gazelle: ## Runs gazelle to update WORKSPACE.
@$(call run,//:gazelle update-repos -from_file=go.mod -prune)
.PHONY: gazelle
##
## Canonical build and test targets.
@@ -126,23 +170,23 @@ TOTAL_PARTITIONS ?= 1
PARTITIONS := --test_arg=--partition=$(PARTITION) --test_arg=--total_partitions=$(TOTAL_PARTITIONS)
runsc: ## Builds the runsc binary.
@$(call submake,build OPTIONS="-c opt" TARGETS="//runsc")
@$(call build,-c opt //runsc)
.PHONY: runsc
debian: ## Builds the debian packages.
@$(call submake,build OPTIONS="-c opt" TARGETS="//debian:debian")
@$(call build,-c opt //debian:debian)
.PHONY: debian
smoke-tests: ## Runs a simple smoke test after build runsc.
@$(call submake,run DOCKER_PRIVILEGED="" ARGS="--alsologtostderr --network none --debug --TESTONLY-unsafe-nonroot=true --rootless do true")
@$(call run,//runsc,--alsologtostderr --network none --debug --TESTONLY-unsafe-nonroot=true --rootless do true)
.PHONY: smoke-tests
fuse-tests:
@$(call submake,test OPTIONS="--test_tag_filters fuse $(PARTITIONS)" TARGETS="test/fuse/...")
@$(call test,--test_tag_filters=fuse $(PARTITIONS) test/fuse/...)
.PHONY: fuse-tests
unit-tests: ## Local package unit tests in pkg/..., runsc/, tools/.., etc.
@$(call submake,test TARGETS="pkg/... runsc/... tools/...")
@$(call test,pkg/... runsc/... tools/...)
.PHONY: unit-tests
tests: ## Runs all unit tests and syscall tests.
@@ -158,101 +202,92 @@ network-tests: ## Run all networking integration tests.
network-tests: iptables-tests packetdrill-tests packetimpact-tests
.PHONY: network-tests
# Standard integration targets.
INTEGRATION_TARGETS := //test/image:image_test //test/e2e:integration_test
syscall-%-tests:
@$(call submake,test OPTIONS="--test_tag_filters runsc_$* $(PARTITIONS)" TARGETS="test/syscalls/...")
@$(call test,--test_tag_filters=runsc_$* $(PARTITIONS) test/syscalls/...)
syscall-native-tests:
@$(call submake,test OPTIONS="--test_tag_filters native $(PARTITIONS)" TARGETS="test/syscalls/...")
@$(call test,--test_tag_filters=native $(PARTITIONS) test/syscalls/...)
.PHONY: syscall-native-tests
syscall-tests: ## Run all system call tests.
@$(call submake,test OPTIONS="$(PARTITIONS)" TARGETS="test/syscalls/...")
@$(call test,$(PARTITIONS) test/syscalls/...)
%-runtime-tests: load-runtimes_%
@$(call submake,install-runtime)
@$(call submake,test-runtime OPTIONS="--test_timeout=10800" TARGETS="//test/runtimes:$*")
%-runtime-tests: load-runtimes_% $(RUNTIME_BIN)
@$(call install_runtime,) # Ensure flags are cleared.
@$(call test_runtime,--test_timeout=10800 //test/runtimes:$*)
%-runtime-tests_vfs2: load-runtimes_%
@$(call submake,install-runtime RUNTIME="vfs2" ARGS="--vfs2")
@$(call submake,test-runtime RUNTIME="vfs2" OPTIONS="--test_timeout=10800" TARGETS="//test/runtimes:$*")
%-runtime-tests_vfs2: load-runtimes_% $(RUNTIME_BIN)
@$(call install_runtime,--vfs2)
@$(call test_runtime,--test_timeout=10800 //test/runtimes:$*)
do-tests: runsc
@$(call submake,run TARGETS="//runsc" ARGS="--rootless do true")
@$(call submake,run TARGETS="//runsc" ARGS="--rootless -network=none do true")
@$(call submake,sudo TARGETS="//runsc" ARGS="do true")
do-tests:
@$(call run,//runsc,--rootless do true)
@$(call run,//runsc,--rootless -network=none do true)
@$(call sudo,//runsc,do true)
.PHONY: do-tests
simple-tests: unit-tests # Compatibility target.
.PHONY: simple-tests
docker-tests: load-basic-images
@$(call submake,install-runtime RUNTIME="vfs1")
@$(call submake,test-runtime RUNTIME="vfs1" TARGETS="$(INTEGRATION_TARGETS)")
@$(call submake,install-runtime RUNTIME="vfs2" ARGS="--vfs2")
@$(call submake,test-runtime RUNTIME="vfs2" TARGETS="$(INTEGRATION_TARGETS)")
# Standard integration targets.
INTEGRATION_TARGETS := //test/image:image_test //test/e2e:integration_test
docker-tests: load-basic $(RUNTIME_BIN)
@$(call install_runtime,) # Clear flags.
@$(call test_runtime,$(INTEGRATION_TARGETS))
@$(call install_runtime,--vfs2)
@$(call test_runtime,$(INTEGRATION_TARGETS))
.PHONY: docker-tests
overlay-tests: load-basic-images
@$(call submake,install-runtime RUNTIME="overlay" ARGS="--overlay")
@$(call submake,test-runtime RUNTIME="overlay" TARGETS="$(INTEGRATION_TARGETS)")
overlay-tests: load-basic $(RUNTIME_BIN)
@$(call install_runtime,--overlay)
@$(call test_runtime,$(INTEGRATION_TARGETS))
.PHONY: overlay-tests
swgso-tests: load-basic-images
@$(call submake,install-runtime RUNTIME="swgso" ARGS="--software-gso=true --gso=false")
@$(call submake,test-runtime RUNTIME="swgso" TARGETS="$(INTEGRATION_TARGETS)")
swgso-tests: load-basic $(RUNTIME_BIN)
@$(call install_runtime,--software-gso=true --gso=false)
@$(call test_runtime,$(INTEGRATION_TARGETS))
.PHONY: swgso-tests
hostnet-tests: load-basic-images
@$(call submake,install-runtime RUNTIME="hostnet" ARGS="--network=host")
@$(call submake,test-runtime RUNTIME="hostnet" OPTIONS="--test_arg=-checkpoint=false --test_arg=-hostnet=true" TARGETS="$(INTEGRATION_TARGETS)")
hostnet-tests: load-basic $(RUNTIME_BIN)
@$(call install_runtime,--network=host)
@$(call test_runtime,--test_arg=-checkpoint=false --test_arg=-hostnet=true $(INTEGRATION_TARGETS))
.PHONY: hostnet-tests
kvm-tests: load-basic-images
kvm-tests: load-basic $(RUNTIME_BIN)
@(lsmod | grep -E '^(kvm_intel|kvm_amd)') || sudo modprobe kvm
@if ! [[ -w /dev/kvm ]]; then sudo chmod a+rw /dev/kvm; fi
@$(call submake,test TARGETS="//pkg/sentry/platform/kvm:kvm_test")
@$(call submake,install-runtime RUNTIME="kvm" ARGS="--platform=kvm")
@$(call submake,test-runtime RUNTIME="kvm" TARGETS="$(INTEGRATION_TARGETS)")
@if ! test -w /dev/kvm; then sudo chmod a+rw /dev/kvm; fi
@$(call test,//pkg/sentry/platform/kvm:kvm_test)
@$(call install_runtime,--platform=kvm)
@$(call test_runtime,$(INTEGRATION_TARGETS))
.PHONY: kvm-tests
iptables-tests: load-iptables
iptables-tests: load-iptables $(RUNTIME_BIN)
@sudo modprobe iptable_filter
@sudo modprobe ip6table_filter
@$(call submake,test-runtime RUNTIME="runc" TARGETS="//test/iptables:iptables_test")
@$(call submake,install-runtime RUNTIME="iptables" ARGS="--net-raw")
@$(call submake,test-runtime RUNTIME="iptables" TARGETS="//test/iptables:iptables_test")
@$(call test,--test_arg=-runtime=runc $(PARTITIONS) //test/iptables:iptables_test)
@$(call install_runtime,--net-raw)
@$(call test_runtime,//test/iptables:iptables_test)
.PHONY: iptables-tests
# Run the iptables tests with runsc only. Useful for developing to skip runc
# testing.
iptables-runsc-tests: load-iptables
@sudo modprobe iptable_filter
@sudo modprobe ip6table_filter
@$(call submake,install-runtime RUNTIME="iptables" ARGS="--net-raw")
@$(call submake,test-runtime RUNTIME="iptables" TARGETS="//test/iptables:iptables_test")
.PHONY: iptables-runsc-tests
packetdrill-tests: load-packetdrill
@$(call submake,install-runtime RUNTIME="packetdrill")
@$(call submake,test-runtime RUNTIME="packetdrill" TARGETS="$(shell $(MAKE) -s query TARGETS='attr(tags, packetdrill, tests(//...))')")
packetdrill-tests: load-packetdrill $(RUNTIME_BIN)
@$(call install_runtime,) # Clear flags.
@$(call test_runtime,//test/packetdrill:all_tests)
.PHONY: packetdrill-tests
packetimpact-tests: load-packetimpact
packetimpact-tests: load-packetimpact $(RUNTIME_BIN)
@sudo modprobe iptable_filter
@sudo modprobe ip6table_filter
@$(call submake,install-runtime RUNTIME="packetimpact")
@$(call submake,test-runtime OPTIONS="--jobs=HOST_CPUS*3 --local_test_jobs=HOST_CPUS*3" RUNTIME="packetimpact" TARGETS="$(shell $(MAKE) -s query TARGETS='attr(tags, packetimpact, tests(//...))')")
@$(call install_runtime,) # Clear flags.
@$(call test_runtime,--jobs=HOST_CPUS*3 --local_test_jobs=HOST_CPUS*3 //test/packetimpact/tests:all_tests)
.PHONY: packetimpact-tests
# Specific containerd version tests.
containerd-test-%: load-basic_alpine load-basic_python load-basic_busybox load-basic_resolv load-basic_httpd load-basic_ubuntu
@$(call submake,install-runtime RUNTIME="root")
@CONTAINERD_VERSION=$* $(MAKE) -s sudo TARGETS="tools/installers:containerd"
@$(MAKE) -s sudo TARGETS="tools/installers:shim"
@$(MAKE) -s sudo TARGETS="test/root:root_test" ARGS="--runtime=root -test.v"
containerd-test-%: load-basic_alpine load-basic_python load-basic_busybox load-basic_resolv load-basic_httpd load-basic_ubuntu $(RUNTIME_BIN)
@$(call install_runtime,) # Clear flags.
@$(call sudo,tools/installers:containerd,$*)
@$(call sudo,tools/installers:shim)
@$(call sudo,test/root:root_test,--runtime=$(RUNTIME) -test.v)
# Note that we can't run containerd-test-1.1.8 tests here.
#
@@ -290,33 +325,33 @@ BENCHMARKS_PLATFORMS := ptrace
BENCHMARKS_TARGETS := //test/benchmarks/base:startup_test
BENCHMARKS_ARGS := -test.bench=. -pprof-cpu -pprof-heap -pprof-heap -pprof-block
init-benchmark-table: ## Initializes a BigQuery table with the benchmark schema
## (see //tools/bigquery/bigquery.go). If the table alread exists, this is a noop.
$(call submake, run TARGETS=//tools/parsers:parser ARGS="init --project=$(BENCHMARKS_PROJECT) \
--dataset=$(BENCHMARKS_DATASET) --table=$(BENCHMARKS_TABLE)")
init-benchmark-table: ## Initializes a BigQuery table with the benchmark schema.
@$(call run,//tools/parsers:parser,init --project=$(BENCHMARKS_PROJECT) --dataset=$(BENCHMARKS_DATASET) --table=$(BENCHMARKS_TABLE))
.PHONY: init-benchmark-table
benchmark-platforms: load-benchmarks-images ## Runs benchmarks for runc and all given platforms in BENCHMARK_PLATFORMS.
$(foreach PLATFORM,$(BENCHMARKS_PLATFORMS), \
$(call submake,run-benchmark RUNTIME="$(PLATFORM)" ARGS="--platform=$(PLATFORM) --vfs2") && \
$(call submake,run-benchmark RUNTIME="$(PLATFORM)_vfs1" ARGS="--platform=$(PLATFORM)") && \
# $(1) is the runtime name, $(2) are the arguments.
run_benchmark = \
$(call header,BENCHMARK $(1) $(2)); \
if test "$(1)" != "runc"; then $(call install_runtime,--profile $(2)); fi \
@T=$$(mktemp --tmpdir logs.$(RUNTIME).XXXXXX); \
$(call sudo,$(BENCHMARKS_TARGETS) --runtime=$(RUNTIME) $(BENCHMARKS_ARGS) | tee $$T); \
rc=$$?; \
if test $$rc -eq 0 && test "$(BENCHMARKS_UPLOAD)" == "true"; then \
$(call run,tools/parsers:parser parse --debug --file=$$T --runtime=$(RUNTIME) --suite_name=$(BENCHMARKS_SUITE) --project=$(BENCHMARKS_PROJECT) --dataset=$(BENCHMARKS_DATASET) --table=$(BENCHMARKS_TABLE) --official=$(BENCHMARKS_OFFICIAL)); \
fi; \
rm -rf $$T; \
exit $$rc
benchmark-platforms: load-benchmarks ## Runs benchmarks for runc and all given platforms in BENCHMARK_PLATFORMS.
@$(foreach PLATFORM,$(BENCHMARKS_PLATFORMS), \
$(call run_benchmark,$(RUNTIME)+vfs2,$(BENCHMARK_ARGS) --platform=$(PLATFORM) --vfs2) && \
$(call run_benchmark,$(RUNTIME),$(BENCHMARK_ARGS) --platform=$(PLATFORM)) && \
) \
$(call submake, run-benchmark RUNTIME="runc")
$(call run-benchmark,runc)
.PHONY: benchmark-platforms
run-benchmark: load-benchmarks-images ## Runs single benchmark and optionally sends data to BigQuery.
@if [[ "$(RUNTIME)" != "runc" ]]; then $(call submake,install-runtime ARGS="$(ARGS) --profile"); fi
@T=$$(mktemp --tmpdir logs.$(RUNTIME).XXXXXX); \
$(call submake,sudo TARGETS="$(BENCHMARKS_TARGETS)" ARGS="--runtime=$(RUNTIME) $(BENCHMARKS_ARGS) | tee $$T"); \
rc=$$?; \
if [[ $$rc -eq 0 ]] && [[ "$(BENCHMARKS_UPLOAD)" == "true" ]]; then \
$(call submake,run TARGETS="tools/parsers:parser" ARGS="parse --debug --file=$$T \
--runtime=$(RUNTIME) --suite_name=$(BENCHMARKS_SUITE) \
--project=$(BENCHMARKS_PROJECT) --dataset=$(BENCHMARKS_DATASET) \
--table=$(BENCHMARKS_TABLE) --official=$(BENCHMARKS_OFFICIAL)"); \
fi; \
rm -rf $$T; \
exit $$rc
run-benchmark: load-benchmarks ## Runs single benchmark and optionally sends data to BigQuery.
@$(call run_benchmark,$(RUNTIME),$(BENCHMARK_ARGS))
.PHONY: run-benchmark
##
@@ -336,7 +371,7 @@ WEBSITE_PROJECT := gvisordev
WEBSITE_REGION := us-central1
website-build: load-jekyll ## Build the site image locally.
@$(call submake,run TARGETS="//website:website" ARGS="$(WEBSITE_IMAGE)")
@$(call run,//website:website $(WEBSITE_IMAGE))
.PHONY: website-build
website-server: website-build ## Run a local server for development.
@@ -362,17 +397,17 @@ website-deploy: website-push ## Deploy a new version of the website.
## RELEASE_NAME - The name of the release in the proper format (needed for tag).
## RELEASE_NOTES - The file containing release notes (needed for tag).
##
RELEASE_ROOT := $(CURDIR)/repo
RELEASE_KEY := repo.key
RELEASE_NIGHTLY := false
RELEASE_COMMIT :=
RELEASE_NAME :=
RELEASE_NOTES :=
RELEASE_ROOT := $(CURDIR)/repo
RELEASE_KEY := repo.key
RELEASE_NIGHTLY := false
RELEASE_COMMIT :=
RELEASE_NAME :=
RELEASE_NOTES :=
GPG_TEST_OPTIONS := $(shell if gpg --pinentry-mode loopback --version >/dev/null 2>&1; then echo --pinentry-mode loopback; fi)
$(RELEASE_KEY):
@echo "WARNING: Generating a key for testing ($@); don't use this."
T=$$(mktemp --tmpdir keyring.XXXXXX); \
@T=$$(mktemp --tmpdir keyring.XXXXXX); \
C=$$(mktemp --tmpdir config.XXXXXX); \
echo Key-Type: DSA >> $$C && \
echo Key-Length: 1024 >> $$C && \
@@ -386,11 +421,11 @@ $(RELEASE_KEY):
release: $(RELEASE_KEY) ## Builds a release.
@mkdir -p $(RELEASE_ROOT)
@T=$$(mktemp -d --tmpdir release.XXXXXX); \
$(call submake,copy TARGETS="//runsc:runsc" DESTINATION=$$T) && \
$(call submake,copy TARGETS="//shim/v1:gvisor-containerd-shim" DESTINATION=$$T) && \
$(call submake,copy TARGETS="//shim/v2:containerd-shim-runsc-v1" DESTINATION=$$T) && \
$(call submake,copy TARGETS="//debian:debian" DESTINATION=$$T) && \
@export T=$$(mktemp -d --tmpdir release.XXXXXX); \
$(call copy,//runsc:runsc,$$T) && \
$(call copy,//shim/v1:gvisor-containerd-shim,$$T) && \
$(call copy,//shim/v2:containerd-shim-runsc-v1,$$T) && \
$(call copy,//debian:debian,$$T) && \
NIGHTLY=$(RELEASE_NIGHTLY) tools/make_release.sh $(RELEASE_KEY) $(RELEASE_ROOT) $$T/*; \
rc=$$?; rm -rf $$T; exit $$rc
.PHONY: release
@@ -398,74 +433,3 @@ release: $(RELEASE_KEY) ## Builds a release.
tag: ## Creates and pushes a release tag.
@tools/tag_release.sh "$(RELEASE_COMMIT)" "$(RELEASE_NAME)" "$(RELEASE_NOTES)"
.PHONY: tag
##
## Development helpers and tooling.
##
## These targets faciliate local development by automatically
## installing and configuring a runtime. Several variables may
## be used here to tweak the installation:
## RUNTIME - The name of the installed runtime (default: branch).
## RUNTIME_DIR - Where the runtime will be installed (default: temporary directory with the $RUNTIME).
## RUNTIME_BIN - The runtime binary (default: $RUNTIME_DIR/runsc).
## RUNTIME_LOG_DIR - The logs directory (default: $RUNTIME_DIR/logs).
## RUNTIME_LOGS - The log pattern (default: $RUNTIME_LOG_DIR/runsc.log.%TEST%.%TIMESTAMP%.%COMMAND%).
##
ifeq (,$(BRANCH_NAME))
RUNTIME := runsc
RUNTIME_DIR := $(shell dirname $(shell mktemp -u))/$(RUNTIME)
else
RUNTIME := $(BRANCH_NAME)
RUNTIME_DIR := $(shell dirname $(shell mktemp -u))/$(RUNTIME)
endif
RUNTIME_BIN := $(RUNTIME_DIR)/runsc
RUNTIME_LOG_DIR := $(RUNTIME_DIR)/logs
RUNTIME_LOGS := $(RUNTIME_LOG_DIR)/runsc.log.%TEST%.%TIMESTAMP%.%COMMAND%
dev: ## Installs a set of local runtimes. Requires sudo.
@$(call submake,refresh)
@$(call submake,configure RUNTIME_NAME="$(RUNTIME)" ARGS="--net-raw")
@$(call submake,configure RUNTIME_NAME="$(RUNTIME)-d" ARGS="--net-raw --debug --strace --log-packets")
@$(call submake,configure RUNTIME_NAME="$(RUNTIME)-p" ARGS="--net-raw --profile")
@$(call submake,configure RUNTIME_NAME="$(RUNTIME)-vfs2-d" ARGS="--net-raw --debug --strace --log-packets --vfs2")
@sudo systemctl restart docker
.PHONY: dev
refresh: ## Refreshes the runtime binary (for development only). Must have called 'dev' or 'install-runtime' first.
@mkdir -p "$(RUNTIME_DIR)"
@$(call submake,copy TARGETS=runsc DESTINATION="$(RUNTIME_BIN)")
.PHONY: refresh
install-runtime: ## Installs the runtime for testing. Requires sudo.
@$(call submake,refresh)
@$(call submake,configure RUNTIME_NAME="$(RUNTIME)" ARGS="$(ARGS) --TESTONLY-test-name-env=RUNSC_TEST_NAME")
@sudo systemctl restart docker
@if [[ -f /etc/docker/daemon.json ]]; then \
sudo chmod 0755 /etc/docker && \
sudo chmod 0644 /etc/docker/daemon.json; \
fi
.PHONY: install-runtime
install-debug-runtime: ## Installs the runtime for debugging. Requires sudo.
@$(call submake,install-runtime ARGS="--debug --strace --log-packets $(ARGS)")
.PHONY: install-debug-runtime
configure: ## Configures a single runtime. Requires sudo. Typically called from dev or install-runtime.
@sudo sudo "$(RUNTIME_BIN)" install --experimental=true --runtime="$(RUNTIME_NAME)" -- --debug-log "$(RUNTIME_LOGS)" $(ARGS)
@echo -e "$(INFO) Installed runtime \"$(RUNTIME)\" @ $(RUNTIME_BIN)"
@echo -e "$(INFO) Logs are in: $(RUNTIME_LOG_DIR)"
@sudo rm -rf "$(RUNTIME_LOG_DIR)" && mkdir -p "$(RUNTIME_LOG_DIR)"
.PHONY: configure
test-runtime: ## A convenient wrapper around test that provides the runtime argument. Target must still be provided.
@$(call submake,test OPTIONS="$(OPTIONS) --test_arg=--runtime=$(RUNTIME) $(PARTITIONS)")
.PHONY: test-runtime
nogo: ## Surfaces all nogo findings.
@$(call submake,build OPTIONS="--build_tag_filters nogo" TARGETS="//...")
@$(call submake,run TARGETS="//tools/github" ARGS="$(foreach dir,$(BUILD_ROOTS),-path=$(CURDIR)/$(dir)) -dry-run nogo")
.PHONY: nogo
gazelle: ## Runs gazelle to update WORKSPACE.
@$(call submake,run TARGETS="//:gazelle" ARGS="update-repos -from_file=go.mod -prune")
.PHONY: gazelle
images/BUILD
-10
View File
@@ -1,11 +1 @@
package(licenses = ["notice"])
# The images filegroup is definitely not a hermetic target, and requires Make
# to do anything meaningful with. However, this will be slurped up and used by
# the tools/installer/images.sh installer, which will ensure that all required
# images are available locally when running vm_tests.
filegroup(
name = "images",
srcs = glob(["**"]),
visibility = ["//tools/installers:__pkg__"],
)
images/Makefile
-107
View File
@@ -1,107 +0,0 @@
#!/usr/bin/make -f
# Copyright 2018 The gVisor Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ARCH is the architecture used for the build. This may be overriden at the
# command line in order to perform a cross-build (in a limited capacity).
ARCH := $(shell uname -m)
# Note that the image prefixes used here must match the image mangling in
# runsc/testutil.MangleImage. Names are mangled in this way to ensure that all
# tests are using locally-defined images (that are consistent and idempotent).
REMOTE_IMAGE_PREFIX ?= gcr.io/gvisor-presubmit
LOCAL_IMAGE_PREFIX ?= gvisor.dev/images
ALL_IMAGES := $(subst /,_,$(subst ./,,$(shell find . -name Dockerfile -o -name Dockerfile.$(ARCH) | xargs -n 1 dirname | uniq)))
ifneq ($(ARCH),$(shell uname -m))
DOCKER_PLATFORM_ARGS := --platform=$(ARCH)
else
DOCKER_PLATFORM_ARGS :=
endif
list-all-images:
@for image in $(ALL_IMAGES); do echo $${image}; done
.PHONY: list-build-images
# Handy wrapper to allow load-all-images, push-all-images, etc.
%-all-images:
@$(MAKE) -s $(patsubst %,$*-%,$(ALL_IMAGES))
load-all-images:
@$(MAKE) -s $(patsubst %,load-%,$(ALL_IMAGES))
# Handy wrapper to load specified "groups", e.g. load-basic-images, etc.
load-%-images:
@$(MAKE) -s $(patsubst %,load-%,$(subst /,_,$(subst ./,,$(shell find ./$* -name Dockerfile -exec dirname {} \;))))
# tag is a function that returns the tag name, given an image.
#
# The tag constructed is used to memoize the image generated (see README.md).
# This scheme is used to enable aggressive caching in a central repository, but
# ensuring that images will always be sourced using the local files if there
# are changes.
path = $(subst _,/,$(1))
dockerfile = $$(if [ -f "$(call path,$(1))/Dockerfile.$(ARCH)" ]; then echo Dockerfile.$(ARCH); else echo Dockerfile; fi)
tag = $(shell find $(call path,$(1)) -type f -print | sort | xargs -n 1 sha256sum | sha256sum - | cut -c 1-16)
remote_image = $(REMOTE_IMAGE_PREFIX)/$(subst _,/,$(1))_$(ARCH):$(call tag,$(1))
local_image = $(LOCAL_IMAGE_PREFIX)/$(subst _,/,$(1))
# rebuild builds the image locally. Only the "remote" tag will be applied. Note
# we need to explicitly repull the base layer in order to ensure that the
# architecture is correct. Note that we use the term "rebuild" here to avoid
# conflicting with the bazel "build" terminology, which is used elsewhere.
rebuild-%: FROM=$(shell grep FROM "$(call path,$*)/$(call dockerfile,$*)" | cut -d' ' -f2)
rebuild-%: register-cross
@if ! [ -f "$(call path,$*)/$(call dockerfile,$*)" ]; then \
(echo "ERROR: Dockerfile for $* not found (is it available for $(ARCH)?)." >&2 && exit 1); \
fi
$(foreach IMAGE,$(FROM),docker pull $(DOCKER_PLATFORM_ARGS) $(IMAGE) &&) \
T=$$(mktemp -d) && cp -a $(call path,$*)/* $$T && \
docker build $(DOCKER_PLATFORM_ARGS) \
-f "$$T/$(call dockerfile,$*)" \
-t "$(call remote_image,$*)" \
$$T && \
rm -rf $$T
# pull will check the "remote" image and pull if necessary. If the remote image
# must be pulled, then it will tag with the latest local target. Note that pull
# may fail if the remote image is not available.
pull-%:
docker pull $(DOCKER_PLATFORM_ARGS) $(call remote_image,$*)
# load will either pull the "remote" or build it locally. This is the preferred
# entrypoint, as it should never fail. The local tag should always be set after
# this returns (either by the pull or the build).
load-%:
$(MAKE) -s pull-$* || $(MAKE) -s rebuild-$*
docker tag $(call remote_image,$*) $(call local_image,$*)
# push pushes the remote image, after either pulling (to validate that the tag
# already exists) or building manually.
push-%: load-%
docker push $(call remote_image,$*)
# register-cross registers the necessary qemu binaries for cross-compilation.
# This may be used by any target that may execute containers that are not the
# native format.
register-cross:
ifneq ($(ARCH),$(shell uname -m))
ifeq (,$(wildcard /proc/sys/fs/binfmt_misc/qemu-*))
docker run --rm --privileged multiarch/qemu-user-static --reset --persistent yes
else
@true # Already registered.
endif
else
@true # No cross required.
endif
.PHONY: register-cross
images/default/Dockerfile
+1 -1
View File
@@ -6,7 +6,7 @@ RUN pip install --no-cache-dir pycparser
RUN dnf install -y bazel3
# Install gcloud.
RUN curl https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-289.0.0-linux-x86_64.tar.gz | \
tar zxvf - google-cloud-sdk && \
tar zxf - google-cloud-sdk && \
google-cloud-sdk/install.sh && \
ln -s /google-cloud-sdk/bin/gcloud /usr/bin/gcloud
# Install Docker client for the website build.
test/packetdrill/BUILD
+9
View File
@@ -38,6 +38,15 @@ packetdrill_test(
scripts = ["tcp_defer_accept_timeout.pkt"],
)
test_suite(
name = "all_tests",
tags = [
"manual",
"packetdrill",
],
tests = existing_rules(),
)
bzl_library(
name = "defs_bzl",
srcs = ["defs.bzl"],
test/packetimpact/tests/BUILD
+9
View File
@@ -383,3 +383,12 @@ validate_all_tests()
expect_netstack_failure = hasattr(t, "expect_netstack_failure"),
num_duts = t.num_duts if hasattr(t, "num_duts") else 1,
) for t in ALL_TESTS]
test_suite(
name = "all_tests",
tags = [
"manual",
"packetimpact",
],
tests = existing_rules(),
)
test/root/BUILD
+1 -13
View File
@@ -1,5 +1,4 @@
load("//tools:defs.bzl", "go_library", "go_test")
load("//tools/vm:defs.bzl", "vm_test")
package(licenses = ["notice"])
@@ -24,12 +23,8 @@ go_test(
],
library = ":root",
tags = [
# Requires docker and runsc to be configured before the test runs.
# Also, the test needs to be run as root. Note that below, the
# root_vm_test relies on the default runtime 'runsc' being installed by
# the default installer.
"manual",
"local",
"manual",
],
visibility = ["//:sandbox"],
deps = [
@@ -46,10 +41,3 @@ go_test(
"@org_golang_x_sys//unix:go_default_library",
],
)
vm_test(
name = "root_vm_test",
size = "large",
shard_count = 1,
targets = [":root_test"],
)
tools/bazel.mk
+118 -121
View File
@@ -14,22 +14,36 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Make hacks.
EMPTY :=
SPACE := $(EMPTY) $(EMPTY)
##
## Docker options.
##
## This file supports targets that wrap bazel in a running Docker
## container to simplify development. Some options are available to
## control the behavior of this container:
##
## USER - The in-container user.
## DOCKER_RUN_OPTIONS - Options for the container (default: --privileged, required for tests).
## DOCKER_NAME - The container name (default: gvisor-bazel-HASH).
## DOCKER_PRIVILEGED - Docker privileged flags (default: --privileged).
## BAZEL_CACHE - The bazel cache directory (default: detected).
## GCLOUD_CONFIG - The gcloud config directory (detect: detected).
## DOCKER_SOCKET - The Docker socket (default: detected).
##
## To opt out of these wrappers, set DOCKER_BUILD=false.
DOCKER_BUILD := true
ifeq ($(DOCKER_BUILD),true)
-include bazel-server
endif
# See base Makefile.
SHELL=/bin/bash -o pipefail
BRANCH_NAME := $(shell (git branch --show-current 2>/dev/null || \
git rev-parse --abbrev-ref HEAD 2>/dev/null) | \
xargs -n 1 basename 2>/dev/null)
git rev-parse --abbrev-ref HEAD 2>/dev/null) | \
xargs -n 1 basename 2>/dev/null)
BUILD_ROOTS := bazel-bin/ bazel-out/
# Bazel container configuration (see below).
USER := $(shell whoami)
HASH := $(shell readlink -m $(CURDIR) | md5sum | cut -c1-8)
BUILDER_BASE := gvisor.dev/images/default
BUILDER_IMAGE := gvisor.dev/images/builder
BUILDER_NAME := gvisor-builder-$(HASH)
DOCKER_NAME := gvisor-bazel-$(HASH)
DOCKER_PRIVILEGED := --privileged
@@ -38,28 +52,43 @@ GCLOUD_CONFIG := $(shell readlink -m ~/.config/gcloud/)
DOCKER_SOCKET := /var/run/docker.sock
DOCKER_CONFIG := /etc/docker
# Bazel flags.
BAZEL := bazel $(STARTUP_OPTIONS)
BASE_OPTIONS := --color=no --curses=no
##
## Bazel helpers.
##
## Bazel will be run with standard flags. You can specify the following flags
## to control which flags are passed:
##
## STARTUP_OPTIONS - Startup options passed to Bazel.
## BAZEL_CONFIG - A bazel config file.
##
STARTUP_OPTIONS :=
BAZEL_CONFIG :=
BAZEL := bazel $(STARTUP_OPTIONS)
BASE_OPTIONS := --color=no --curses=no
ifneq (,$(BAZEL_CONFIG))
BASE_OPTIONS += --config=$(BAZEL_CONFIG)
endif
TEST_OPTIONS := $(BASE_OPTIONS) \
--test_output=errors \
--keep_going \
--verbose_failures=true \
--build_event_json_file=.build_events.json
# Basic options.
UID := $(shell id -u ${USER})
GID := $(shell id -g ${USER})
USERADD_OPTIONS :=
FULL_DOCKER_RUN_OPTIONS := $(DOCKER_RUN_OPTIONS)
FULL_DOCKER_RUN_OPTIONS += --user $(UID):$(GID)
FULL_DOCKER_RUN_OPTIONS += --entrypoint ""
FULL_DOCKER_RUN_OPTIONS += --init
FULL_DOCKER_RUN_OPTIONS += -v "$(BAZEL_CACHE):$(BAZEL_CACHE)"
FULL_DOCKER_RUN_OPTIONS += -v "$(GCLOUD_CONFIG):$(GCLOUD_CONFIG)"
FULL_DOCKER_RUN_OPTIONS += -v "/tmp:/tmp"
FULL_DOCKER_EXEC_OPTIONS := --user $(UID):$(GID)
FULL_DOCKER_EXEC_OPTIONS += --interactive
ifeq (true,$(shell [[ -t 0 ]] && echo true))
FULL_DOCKER_EXEC_OPTIONS += --tty
DOCKER_RUN_OPTIONS :=
DOCKER_RUN_OPTIONS += --user $(UID):$(GID)
DOCKER_RUN_OPTIONS += --entrypoint ""
DOCKER_RUN_OPTIONS += --init
DOCKER_RUN_OPTIONS += -v "$(BAZEL_CACHE):$(BAZEL_CACHE)"
DOCKER_RUN_OPTIONS += -v "$(GCLOUD_CONFIG):$(GCLOUD_CONFIG)"
DOCKER_RUN_OPTIONS += -v "/tmp:/tmp"
DOCKER_EXEC_OPTIONS := --user $(UID):$(GID)
DOCKER_EXEC_OPTIONS += --interactive
ifeq (true,$(shell test -t 0 && echo true))
DOCKER_EXEC_OPTIONS += --tty
endif
# Add basic UID/GID options.
@@ -83,86 +112,75 @@ endif
# Add docker passthrough options.
ifneq ($(DOCKER_PRIVILEGED),)
FULL_DOCKER_RUN_OPTIONS += -v "$(DOCKER_SOCKET):$(DOCKER_SOCKET)"
FULL_DOCKER_RUN_OPTIONS += -v "$(DOCKER_CONFIG):$(DOCKER_CONFIG)"
FULL_DOCKER_RUN_OPTIONS += $(DOCKER_PRIVILEGED)
FULL_DOCKER_EXEC_OPTIONS += $(DOCKER_PRIVILEGED)
DOCKER_RUN_OPTIONS += -v "$(DOCKER_SOCKET):$(DOCKER_SOCKET)"
DOCKER_RUN_OPTIONS += -v "$(DOCKER_CONFIG):$(DOCKER_CONFIG)"
DOCKER_RUN_OPTIONS += $(DOCKER_PRIVILEGED)
DOCKER_EXEC_OPTIONS += $(DOCKER_PRIVILEGED)
DOCKER_GROUP := $(shell stat -c '%g' $(DOCKER_SOCKET))
ifneq ($(GID),$(DOCKER_GROUP))
USERADD_OPTIONS += --groups $(DOCKER_GROUP)
GROUPADD_DOCKER += groupadd --gid $(DOCKER_GROUP) --non-unique docker-$(HASH) &&
FULL_DOCKER_RUN_OPTIONS += --group-add $(DOCKER_GROUP)
DOCKER_RUN_OPTIONS += --group-add $(DOCKER_GROUP)
endif
endif
# Add KVM passthrough options.
ifneq (,$(wildcard /dev/kvm))
FULL_DOCKER_RUN_OPTIONS += --device=/dev/kvm
DOCKER_RUN_OPTIONS += --device=/dev/kvm
KVM_GROUP := $(shell stat -c '%g' /dev/kvm)
ifneq ($(GID),$(KVM_GROUP))
USERADD_OPTIONS += --groups $(KVM_GROUP)
GROUPADD_DOCKER += groupadd --gid $(KVM_GROUP) --non-unique kvm-$(HASH) &&
FULL_DOCKER_RUN_OPTIONS += --group-add $(KVM_GROUP)
DOCKER_RUN_OPTIONS += --group-add $(KVM_GROUP)
endif
endif
bazel-image: load-default
@if docker ps --all | grep $(BUILDER_NAME); then docker rm -f $(BUILDER_NAME); fi
docker run --user 0:0 --entrypoint "" --name $(BUILDER_NAME) \
$(BUILDER_BASE) \
sh -c "$(GROUPADD_DOCKER) \
$(USERADD_DOCKER) \
if [[ -e /dev/kvm ]]; then chmod a+rw /dev/kvm; fi"
docker commit $(BUILDER_NAME) $(BUILDER_IMAGE)
@docker rm -f $(BUILDER_NAME)
.PHONY: bazel-image
##
## Bazel helpers.
##
## This file supports targets that wrap bazel in a running Docker
## container to simplify development. Some options are available to
## control the behavior of this container:
## USER - The in-container user.
## DOCKER_RUN_OPTIONS - Options for the container (default: --privileged, required for tests).
## DOCKER_NAME - The container name (default: gvisor-bazel-HASH).
## BAZEL_CACHE - The bazel cache directory (default: detected).
## GCLOUD_CONFIG - The gcloud config directory (detect: detected).
## DOCKER_SOCKET - The Docker socket (default: detected).
##
bazel-server-start: bazel-image ## Starts the bazel server.
@mkdir -p $(BAZEL_CACHE)
@mkdir -p $(GCLOUD_CONFIG)
@if docker ps --all | grep $(DOCKER_NAME); then docker rm -f $(DOCKER_NAME); fi
# This command runs a bazel server, and the container sticks around
# until the bazel server exits. This should ensure that it does not
# exit in the middle of running a build, but also it won't stick around
# forever. The build commands wrap around an appropriate exec into the
# container in order to perform work via the bazel client.
docker run -d --rm --name $(DOCKER_NAME) \
-v "$(CURDIR):$(CURDIR)" \
--workdir "$(CURDIR)" \
$(FULL_DOCKER_RUN_OPTIONS) \
$(BUILDER_IMAGE) \
sh -c "tail -f --pid=\$$($(BAZEL) info server_pid) /dev/null"
.PHONY: bazel-server-start
# Top-level functions.
#
# This command runs a bazel server, and the container sticks around
# until the bazel server exits. This should ensure that it does not
# exit in the middle of running a build, but also it won't stick around
# forever. The build commands wrap around an appropriate exec into the
# container in order to perform work via the bazel client.
ifeq ($(DOCKER_BUILD),true)
wrapper = docker exec $(DOCKER_EXEC_OPTIONS) $(DOCKER_NAME) $(1)
else
wrapper = $(1)
endif
bazel-shutdown: ## Shuts down a running bazel server.
@docker exec $(FULL_DOCKER_EXEC_OPTIONS) $(DOCKER_NAME) $(BAZEL) shutdown; \
rc=$$?; docker kill $(DOCKER_NAME) || [[ $$rc -ne 0 ]]
@$(call wrapper,$(BAZEL) shutdown)
.PHONY: bazel-shutdown
bazel-alias: ## Emits an alias that can be used within the shell.
@echo "alias bazel='docker exec $(FULL_DOCKER_EXEC_OPTIONS) $(DOCKER_NAME) bazel'"
@echo "alias bazel='$(call wrapper,$(BAZEL))'"
.PHONY: bazel-alias
bazel-server: ## Ensures that the server exists. Used as an internal target.
@docker exec $(FULL_DOCKER_EXEC_OPTIONS) $(DOCKER_NAME) true >&2 || $(MAKE) bazel-server-start >&2
.PHONY: bazel-server
bazel-image: load-default ## Ensures that the local builder exists.
@$(call header,DOCKER BUILD)
@docker rm -f $(BUILDER_NAME) 2>/dev/null || true
@docker run --user 0:0 --entrypoint "" --name $(BUILDER_NAME) gvisor.dev/images/default \
sh -c "$(GROUPADD_DOCKER) $(USERADD_DOCKER) if test -e /dev/kvm; then chmod a+rw /dev/kvm; fi"
@docker commit $(BUILDER_NAME) gvisor.dev/images/builder
.PHONY: bazel-image
# build_cmd builds the given targets in the bazel-server container.
build_cmd = docker exec $(FULL_DOCKER_EXEC_OPTIONS) $(DOCKER_NAME) sh -o pipefail -c \
'$(BAZEL) build $(BASE_OPTIONS) $(OPTIONS) "$(TARGETS)"'
ifeq (,$(findstring $(DOCKER_NAME),$(shell docker ps 2>/dev/null)))
bazel-server: bazel-image ## Ensures that the server exists.
@$(call header,DOCKER RUN)
@docker rm -f $(DOCKER_NAME) 2>/dev/null || true
@mkdir -p $(GCLOUD_CONFIG)
@mkdir -p $(BAZEL_CACHE)
@docker run -d --rm --name $(DOCKER_NAME) \
-v "$(CURDIR):$(CURDIR)" \
--workdir "$(CURDIR)" \
$(DOCKER_RUN_OPTIONS) \
gvisor.dev/images/builder \
sh -c "set -x; tail -f --pid=\$$($(BAZEL) info server_pid) /dev/null"
else
bazel-server:
@
endif
.PHONY: bazel-server
# build_paths extracts the built binary from the bazel stderr output.
#
@@ -172,49 +190,28 @@ build_cmd = docker exec $(FULL_DOCKER_EXEC_OPTIONS) $(DOCKER_NAME) sh -o pipefai
# command here? Yikes, let's just stick with the ugly shell pipeline.
#
# The last line is used to prevent terminal shenanigans.
build_paths = command_line=$$( $(build_cmd) 2>&1 \
| grep -A1 -E '^Target' \
| grep -E '^ ($(subst $(SPACE),|,$(BUILD_ROOTS)))' \
| sed "s/ /\n/g" \
| strings -n 10 \
| awk '{$$1=$$1};1' \
| xargs -n 1 -I {} readlink -f "{}" \
| xargs -n 1 -I {} echo "$(1)" ) && \
(set -xeuo pipefail; eval $${command_line})
build_paths = \
$(call wrapper,$(BAZEL) build $(BASE_OPTIONS) $(1)) 2>&1 \
| tee /proc/self/fd/2 \
| grep -A1 -E '^Target' \
| grep -E '^ ($(subst $(SPACE),|,$(BUILD_ROOTS)))' \
| sed "s/ /\n/g" \
| strings -n 10 \
| awk '{$$1=$$1};1' \
| xargs -n 1 -I {} readlink -f "{}" \
| xargs -n 1 -I {} bash -c 'set -xeuo pipefail; $(2)'
build: bazel-server
@$(call build_cmd)
.PHONY: build
clean = $(call header,CLEAN) && $(call wrapper,$(BAZEL) clean)
build = $(call header,BUILD $(1)) && $(call build_paths,$(1),echo {})
copy = $(call header,COPY $(1) $(2)) && $(call build_paths,$(1),cp -fa {} $(2))
run = $(call header,RUN $(1) $(2)) && $(call build_paths,$(1),{} $(2))
sudo = $(call header,SUDO $(1) $(2)) && $(call build_paths,$(1),sudo -E {} $(2))
test = $(call header,TEST $(1)) && $(call wrapper,$(BAZEL) test $(TEST_OPTIONS) $(1))
copy: bazel-server
ifeq (,$(DESTINATION))
$(error Destination not provided.)
endif
@$(call build_paths,cp -fa {} $(DESTINATION))
run: bazel-server
@$(call build_paths,{} $(ARGS))
.PHONY: run
sudo: bazel-server
@$(call build_paths,sudo -E {} $(ARGS))
.PHONY: sudo
test: bazel-server
@docker exec $(FULL_DOCKER_EXEC_OPTIONS) $(DOCKER_NAME) \
$(BAZEL) test $(BASE_OPTIONS) \
--test_output=errors --keep_going --verbose_failures=true \
--build_event_json_file=.build_events.json \
$(OPTIONS) $(TARGETS)
.PHONY: test
testlogs:
@cat .build_events.json | jq -r \
'select(.testSummary?.overallStatus? | tostring | test("(FAILED|FLAKY|TIMEOUT)")) | .testSummary.failed | .[] | .uri' | \
awk -Ffile:// '{print $$2;}'
testlogs: ## Returns the most recent set of test logs.
@if test -f .build_events.json; then \
cat .build_events.json | jq -r \
'select(.testSummary?.overallStatus? | tostring | test("(FAILED|FLAKY|TIMEOUT)")) | .testSummary.failed | .[] | .uri' | \
awk -Ffile:// '{print $$2;}'; \
fi
.PHONY: testlogs
query: bazel-server
@docker exec $(FULL_DOCKER_EXEC_OPTIONS) $(DOCKER_NAME) sh -o pipefail -c \
'$(BAZEL) query $(BASE_OPTIONS) $(OPTIONS) "$(TARGETS)" 2>/dev/null'
.PHONY: query
tools/go_branch.sh
+1 -1
View File
@@ -90,7 +90,7 @@ find . -type f -exec chmod 0644 {} \;
find . -type d -exec chmod 0755 {} \;
# Sync the entire gopath_dir.
rsync --recursive --verbose --delete --exclude .git -L "${gopath_dir}/" .
rsync --recursive --delete --exclude .git -L "${gopath_dir}/" .
# Add additional files.
for file in "${othersrc[@]}"; do
tools/images.mk
+165
View File
@@ -0,0 +1,165 @@
#!/usr/bin/make -f
# Copyright 2018 The gVisor Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##
## Docker image targets.
##
## Images used by the tests must also be built and available locally.
## The canonical test targets defined below will automatically load
## relevant images. These can be loaded or built manually via these
## targets.
##
## (*) Note that you may provide an ARCH parameter in order to build
## and load images from an alternate archiecture (using qemu). When
## bazel is run as a server, this has the effect of running an full
## cross-architecture chain, and can produce cross-compiled binaries.
##
# ARCH is the architecture used for the build. This may be overriden at the
# command line in order to perform a cross-build (in a limited capacity).
ARCH := $(shell uname -m)
ifneq ($(ARCH),$(shell uname -m))
DOCKER_PLATFORM_ARGS := --platform=$(ARCH)
else
DOCKER_PLATFORM_ARGS :=
endif
# Note that the image prefixes used here must match the image mangling in
# runsc/testutil.MangleImage. Names are mangled in this way to ensure that all
# tests are using locally-defined images (that are consistent and idempotent).
REMOTE_IMAGE_PREFIX ?= gcr.io/gvisor-presubmit
LOCAL_IMAGE_PREFIX ?= gvisor.dev/images
ALL_IMAGES := $(subst /,_,$(subst images/,,$(shell find images/ -name Dockerfile -o -name Dockerfile.$(ARCH) | xargs -n 1 dirname | uniq)))
SUB_IMAGES := $(foreach image,$(ALL_IMAGES),$(if $(findstring _,$(image)),$(image),))
IMAGE_GROUPS := $(sort $(foreach image,$(SUB_IMAGES),$(firstword $(subst _, ,$(image)))))
define expand_group =
load-$(1): $$(patsubst $(1)_%, load-$(1)_%, $$(filter $(1)_%,$$(ALL_IMAGES)))
@
.PHONY: load-$(1)
push-$(1): $$(patsubst $(1)_%, push-$(1)_%, $$(filter $(1)_%,$$(ALL_IMAGES)))
@
.PHONY: push-$(1)
endef
$(foreach group,$(IMAGE_GROUPS),$(eval $(call expand_group,$(group))))
list-all-images: ## List all images.
@for image in $(ALL_IMAGES); do echo $${image}; done
.PHONY: list-all-images
load-all-images: ## Load all images.
load-all-images: $(patsubst %,load-%,$(ALL_IMAGES))
.PHONY: load-all-images
push-all-images: ## Push all images.
push-all-images: $(patsubst %,push-%,$(ALL_IMAGES))
.PHONY: push-all-images
# path and dockerfile are used to extract the relevant path and dockerfile
# (depending on what's available for the given architecture).
path = images/$(subst _,/,$(1))
dockerfile = $$(if [ -f "$(call path,$(1))/Dockerfile.$(ARCH)" ]; then echo Dockerfile.$(ARCH); else echo Dockerfile; fi)
# The tag construct is used to memoize the image generated (see README.md).
# This scheme is used to enable aggressive caching in a central repository, but
# ensuring that images will always be sourced using the local files.
tag = $(shell cd images && find $(subst _,/,$(1)) -type f | sort | xargs -n 1 sha256sum | sha256sum - | cut -c 1-16)
remote_image = $(REMOTE_IMAGE_PREFIX)/$(subst _,/,$(1))_$(ARCH)
local_image = $(LOCAL_IMAGE_PREFIX)/$(subst _,/,$(1))
# Include all existing images as targets here.
#
# Note that we use a _ for the tag separator, instead of :, as the latter is
# interpreted by Make, unfortunately. tag_expand expands the generic rules to
# tag-specific targets. These is needed to provide sensible targets for load
# below, with caching. Basically, if there is a rule generated here, then the
# load will be skipped. If there is no load generated here, then the default
# rule for load will kick in.
#
# Note that if this rule does not successfully rule, we will simply have
# additional Docker pull commands that run for all images that are already
# pulled. No real harm done.
EXISTING_IMAGES = $(shell docker images --format '{{.Repository}}_{{.Tag}}' | grep -v '<none>')
define existing_image_rule =
loaded0_$(1)=load-$$(1): tag-$$(1) # Already available.
loaded1_$(1)=.PHONY: load-$$(1)
endef
$(foreach image, $(EXISTING_IMAGES), $(eval $(call existing_image_rule,$(image))))
define tag_expand_rule =
$(eval $(loaded0_$(call local_image,$(1))_$(call tag,$(1))))
$(eval $(loaded1_$(call local_image,$(1))_$(call tag,$(1))))
endef
$(foreach image, $(ALL_IMAGES), $(eval $(call tag_expand_rule,$(image))))
# tag tags a local image. This applies both the hash-based tag from above to
# ensure that caching works as expected, as well as the "latest" tag that is
# used by the tests.
local_tag = \
docker tag $(call remote_image,$(1)):$(call tag,$(1)) $(call local_image,$(1)):$(call tag,$(1)) && \
docker tag $(call remote_image,$(1)):$(call tag,$(1)) $(call local_image,$(1))
tag-%: ## Tag a local image.
@$(call local_tag,$*)
# pull forces the image to be pulled.
pull = \
$(call header,PULL $(1)) && \
docker pull $(DOCKER_PLATFORM_ARGS) $(call remote_image,$(1)):$(call tag,$(1)) && \
$(call local_tag,$(1))
pull-%: register-cross ## Force a repull of the image.
@$(call pull,$*)
# rebuild builds the image locally. Only the "remote" tag will be applied. Note
# we need to explicitly repull the base layer in order to ensure that the
# architecture is correct. Note that we use the term "rebuild" here to avoid
# conflicting with the bazel "build" terminology, which is used elsewhere.
rebuild = \
$(call header,REBUILD $(1)) && \
(T=$$(mktemp -d) && cp -a $(call path,$(1))/* $$T && \
$(foreach image,$(shell grep FROM "$(call path,$(1))/$(call dockerfile,$(1))" 2>/dev/null | cut -d' ' -f2),docker pull $(DOCKER_PLATFORM_ARGS) $(image) &&) \
docker build $(DOCKER_PLATFORM_ARGS) \
-f "$$T/$(call dockerfile,$(1))" \
-t "$(call remote_image,$(1)):$(call tag,$(1))" \
$$T && \
rm -rf $$T) && \
$(call local_tag,$(1))
rebuild-%: register-cross ## Force rebuild an image locally.
@$(call rebuild,$*)
# load will either pull the "remote" or build it locally. This is the preferred
# entrypoint, as it should never fail. The local tag should always be set after
# this returns (either by the pull or the build).
load-%: register-cross ## Pull or build an image locally.
@($(call pull,$*)) || ($(call rebuild,$*))
# push pushes the remote image, after either pulling (to validate that the tag
# already exists) or building manually. Note that this generic rule will match
# the fully-expanded remote image tag.
push-%: load-% ## Push a given image.
@docker push $(call remote_image,$*):$(call tag,$*)
# register-cross registers the necessary qemu binaries for cross-compilation.
# This may be used by any target that may execute containers that are not the
# native format. Note that this will only apply on the first execution.
register-cross:
ifneq ($(ARCH),$(shell uname -m))
ifeq (,$(wildcard /proc/sys/fs/binfmt_misc/qemu-*))
@docker run --rm --privileged multiarch/qemu-user-static --reset --persistent yes
else
@
endif
else
@
endif
tools/installers/BUILD
+1 -9
View File
@@ -1,4 +1,4 @@
# Installers for use by the tools/vm_test rules.
# Installers for use by top-level scripts.
package(
default_visibility = ["//:sandbox"],
@@ -13,14 +13,6 @@ sh_binary(
],
)
sh_binary(
name = "images",
srcs = ["images.sh"],
data = [
"//images",
],
)
sh_binary(
name = "master",
srcs = ["master.sh"],
tools/installers/containerd.sh
+1 -1
View File
@@ -16,7 +16,7 @@
set -xeo pipefail
declare -r CONTAINERD_VERSION=${CONTAINERD_VERSION:-1.3.0}
declare -r CONTAINERD_VERSION=${1:-1.3.0}
declare -r CONTAINERD_MAJOR="$(echo ${CONTAINERD_VERSION} | awk -F '.' '{ print $1; }')"
declare -r CONTAINERD_MINOR="$(echo ${CONTAINERD_VERSION} | awk -F '.' '{ print $2; }')"
tools/vm/BUILD
-63
View File
@@ -1,63 +0,0 @@
load("//tools:defs.bzl", "bzl_library", "cc_binary", "gtest")
load("//tools/vm:defs.bzl", "vm_image", "vm_test")
package(
default_visibility = ["//:sandbox"],
licenses = ["notice"],
)
sh_binary(
name = "zone",
srcs = ["zone.sh"],
)
sh_binary(
name = "builder",
srcs = ["build.sh"],
)
sh_binary(
name = "executer",
srcs = ["execute.sh"],
)
cc_binary(
name = "test",
testonly = 1,
srcs = ["test.cc"],
linkstatic = 1,
deps = [
gtest,
"//test/util:test_main",
],
)
vm_image(
name = "ubuntu1604",
family = "ubuntu-1604-lts",
project = "ubuntu-os-cloud",
scripts = [
"//tools/vm/ubuntu1604",
],
)
vm_image(
name = "ubuntu1804",
family = "ubuntu-1804-lts",
project = "ubuntu-os-cloud",
scripts = [
"//tools/vm/ubuntu1804",
],
)
vm_test(
name = "vm_test",
shard_count = 2,
targets = [":test"],
)
bzl_library(
name = "defs_bzl",
srcs = ["defs.bzl"],
visibility = ["//visibility:private"],
)
tools/vm/README.md
-48
View File
@@ -1,48 +0,0 @@
# VM Images & Tests
All commands in this directory require the `gcloud` project to be set.
For example: `gcloud config set project gvisor-kokoro-testing`.
Images can be generated by using the `vm_image` rule. This rule will generate a
binary target that builds an image in an idempotent way, and can be referenced
from other rules.
For example:
```
vm_image(
name = "ubuntu",
project = "ubuntu-1604-lts",
family = "ubuntu-os-cloud",
scripts = [
"script.sh",
"other.sh",
],
)
```
These images can be built manually by executing the target. The output on
`stdout` will be the image id (in the current project).
For example:
```
$ bazel build :ubuntu
```
Images are always named per the hash of all the hermetic input scripts. This
allows images to be memoized quickly and easily.
The `vm_test` rule can be used to execute a command remotely. This is still
under development however, and will likely change over time.
For example:
```
vm_test(
name = "mycommand",
image = ":ubuntu",
targets = [":test"],
)
```
tools/vm/build.sh
-117
View File
@@ -1,117 +0,0 @@
#!/bin/bash
# Copyright 2019 The gVisor Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This script is responsible for building a new GCP image that: 1) has nested
# virtualization enabled, and 2) has been completely set up with the
# image_setup.sh script. This script should be idempotent, as we memoize the
# setup script with a hash and check for that name.
set -eou pipefail
# Parameters.
declare -r USERNAME=${USERNAME:-test}
declare -r IMAGE_PROJECT=${IMAGE_PROJECT:-ubuntu-os-cloud}
declare -r IMAGE_FAMILY=${IMAGE_FAMILY:-ubuntu-1604-lts}
declare -r ZONE=${ZONE:-us-central1-f}
# Random names.
declare -r DISK_NAME=$(mktemp -u disk-XXXXXX | tr A-Z a-z)
declare -r SNAPSHOT_NAME=$(mktemp -u snapshot-XXXXXX | tr A-Z a-z)
declare -r INSTANCE_NAME=$(mktemp -u build-XXXXXX | tr A-Z a-z)
# Hash inputs in order to memoize the produced image.
declare -r SETUP_HASH=$( (echo ${USERNAME} ${IMAGE_PROJECT} ${IMAGE_FAMILY} && cat "$@") | sha256sum - | cut -d' ' -f1 | cut -c 1-16)
declare -r IMAGE_NAME=${IMAGE_FAMILY:-image}-${SETUP_HASH}
# Does the image already exist? Skip the build.
declare -r existing=$(set -x; gcloud compute images list --filter="name=(${IMAGE_NAME})" --format="value(name)")
if ! [[ -z "${existing}" ]]; then
echo "${existing}"
exit 0
fi
# Standard arguments (applies only on script execution).
declare -ar SSH_ARGS=("-o" "ConnectTimeout=60" "--")
# gcloud has path errors; is this a result of being a genrule?
export PATH=${PATH:-/bin:/usr/bin:/usr/local/bin}
# Start a unique instance. Note that this instance will have a unique persistent
# disk as it's boot disk with the same name as the instance.
(set -x; gcloud compute instances create \
--quiet \
--image-project "${IMAGE_PROJECT}" \
--image-family "${IMAGE_FAMILY}" \
--boot-disk-size "200GB" \
--zone "${ZONE}" \
"${INSTANCE_NAME}" >/dev/null)
function cleanup {
(set -x; gcloud compute instances delete --quiet --zone "${ZONE}" "${INSTANCE_NAME}")
}
trap cleanup EXIT
# Wait for the instance to become available (up to 5 minutes).
echo -n "Waiting for ${INSTANCE_NAME}" >&2
declare timeout=300
declare success=0
declare internal=""
declare -r start=$(date +%s)
declare -r end=$((${start}+${timeout}))
while [[ "$(date +%s)" -lt "${end}" ]] && [[ "${success}" -lt 3 ]]; do
echo -n "." >&2
if gcloud compute ssh --zone "${ZONE}" "${USERNAME}"@"${INSTANCE_NAME}" -- true 2>/dev/null; then
success=$((${success}+1))
elif gcloud compute ssh --internal-ip --zone "${ZONE}" "${USERNAME}"@"${INSTANCE_NAME}" -- true 2>/dev/null; then
success=$((${success}+1))
internal="--internal-ip"
fi
done
if [[ "${success}" -eq "0" ]]; then
echo "connect timed out after ${timeout} seconds." >&2
exit 1
else
echo "done." >&2
fi
# Run the install scripts provided.
for arg; do
(set -x; gcloud compute ssh ${internal} \
--zone "${ZONE}" \
"${USERNAME}"@"${INSTANCE_NAME}" -- \
"${SSH_ARGS[@]}" \
sudo bash - <"${arg}" >/dev/null)
done
# Stop the instance; required before creating an image.
(set -x; gcloud compute instances stop --quiet --zone "${ZONE}" "${INSTANCE_NAME}" >/dev/null)
# Create a snapshot of the instance disk.
(set -x; gcloud compute disks snapshot \
--quiet \
--zone "${ZONE}" \
--snapshot-names="${SNAPSHOT_NAME}" \
"${INSTANCE_NAME}" >/dev/null)
# Create the disk image.
(set -x; gcloud compute images create \
--quiet \
--source-snapshot="${SNAPSHOT_NAME}" \
--licenses="https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx" \
"${IMAGE_NAME}" >/dev/null)
# Finish up.
echo "${IMAGE_NAME}"
tools/vm/defs.bzl
-202
View File
@@ -1,202 +0,0 @@
"""Image configuration. See README.md."""
load("//tools:defs.bzl", "default_installer")
# vm_image_builder is a rule that will construct a shell script that actually
# generates a given VM image. Note that this does not _run_ the shell script
# (although it can be run manually). It will be run manually during generation
# of the vm_image target itself. This level of indirection is used so that the
# build system itself only runs the builder once when multiple targets depend
# on it, avoiding a set of races and conflicts.
def _vm_image_builder_impl(ctx):
# Generate a binary that actually builds the image.
builder = ctx.actions.declare_file(ctx.label.name)
script_paths = []
for script in ctx.files.scripts:
script_paths.append(script.short_path)
builder_content = "\n".join([
"#!/bin/bash",
"export ZONE=$(%s)" % ctx.files.zone[0].short_path,
"export USERNAME=%s" % ctx.attr.username,
"export IMAGE_PROJECT=%s" % ctx.attr.project,
"export IMAGE_FAMILY=%s" % ctx.attr.family,
"%s %s" % (ctx.files._builder[0].short_path, " ".join(script_paths)),
"",
])
ctx.actions.write(builder, builder_content, is_executable = True)
# Note that the scripts should only be files, and should not include any
# indirect transitive dependencies. The build script wouldn't work.
return [DefaultInfo(
executable = builder,
runfiles = ctx.runfiles(
files = ctx.files.scripts + ctx.files._builder + ctx.files.zone,
),
)]
vm_image_builder = rule(
attrs = {
"_builder": attr.label(
executable = True,
default = "//tools/vm:builder",
cfg = "host",
),
"username": attr.string(default = "$(whoami)"),
"zone": attr.label(
executable = True,
default = "//tools/vm:zone",
cfg = "host",
),
"family": attr.string(mandatory = True),
"project": attr.string(mandatory = True),
"scripts": attr.label_list(allow_files = True),
},
executable = True,
implementation = _vm_image_builder_impl,
)
# See vm_image_builder above.
def _vm_image_impl(ctx):
# Run the builder to generate our output.
echo = ctx.actions.declare_file(ctx.label.name)
resolved_inputs, argv, runfiles_manifests = ctx.resolve_command(
command = "\n".join([
"set -e",
"image=$(%s)" % ctx.files.builder[0].path,
"echo -ne \"#!/bin/bash\\necho ${image}\\n\" > %s" % echo.path,
"chmod 0755 %s" % echo.path,
]),
tools = [ctx.attr.builder],
)
ctx.actions.run_shell(
tools = resolved_inputs,
outputs = [echo],
progress_message = "Building image...",
execution_requirements = {"local": "true"},
command = argv,
input_manifests = runfiles_manifests,
)
# Return just the echo command. All of the builder runfiles have been
# resolved and consumed in the generation of the trivial echo script.
return [DefaultInfo(executable = echo)]
_vm_image_test = rule(
attrs = {
"builder": attr.label(
executable = True,
cfg = "host",
),
},
test = True,
implementation = _vm_image_impl,
)
def vm_image(name, **kwargs):
vm_image_builder(
name = name + "_builder",
**kwargs
)
_vm_image_test(
name = name,
builder = ":" + name + "_builder",
tags = [
"local",
"manual",
],
)
def _vm_test_impl(ctx):
runner = ctx.actions.declare_file("%s-executer" % ctx.label.name)
# Note that the remote execution case must actually generate an
# intermediate target in order to collect all the relevant runfiles so that
# they can be copied over for remote execution.
runner_content = "\n".join([
"#!/bin/bash",
"export ZONE=$(%s)" % ctx.files.zone[0].short_path,
"export USERNAME=%s" % ctx.attr.username,
"export IMAGE=$(%s)" % ctx.files.image[0].short_path,
"export SUDO=%s" % "true" if ctx.attr.sudo else "false",
"%s %s" % (
ctx.executable.executer.short_path,
" ".join([
target.files_to_run.executable.short_path
for target in ctx.attr.targets
]),
),
"",
])
ctx.actions.write(runner, runner_content, is_executable = True)
# Return with all transitive files.
runfiles = ctx.runfiles(
transitive_files = depset(transitive = [
depset(target.data_runfiles.files)
for target in ctx.attr.targets
if hasattr(target, "data_runfiles")
]),
files = ctx.files.executer + ctx.files.zone + ctx.files.image +
ctx.files.targets,
collect_default = True,
collect_data = True,
)
return [DefaultInfo(executable = runner, runfiles = runfiles)]
_vm_test = rule(
attrs = {
"image": attr.label(
executable = True,
default = "//tools/vm:ubuntu1804",
cfg = "host",
),
"executer": attr.label(
executable = True,
default = "//tools/vm:executer",
cfg = "host",
),
"username": attr.string(default = "$(whoami)"),
"zone": attr.label(
executable = True,
default = "//tools/vm:zone",
cfg = "host",
),
"sudo": attr.bool(default = True),
"machine": attr.string(default = "n1-standard-1"),
"targets": attr.label_list(
mandatory = True,
allow_empty = False,
cfg = "target",
),
},
test = True,
implementation = _vm_test_impl,
)
def vm_test(
installers = None,
**kwargs):
"""Runs the given targets as a remote test.
Args:
installer: Script to run before all targets.
**kwargs: All test arguments. Should include targets and image.
"""
targets = kwargs.pop("targets", [])
if installers == None:
installers = [
"//tools/installers:head",
"//tools/installers:images",
]
targets = installers + targets
if default_installer():
targets = [default_installer()] + targets
_vm_test(
tags = [
"local",
"manual",
],
targets = targets,
local = 1,
**kwargs
)
tools/vm/execute.sh
-160
View File
@@ -1,160 +0,0 @@
#!/bin/bash
# Copyright 2019 The gVisor Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -xeo pipefail
# Required input.
if ! [[ -v IMAGE ]]; then
echo "no image provided: set IMAGE."
exit 1
fi
# Parameters.
declare -r USERNAME=${USERNAME:-test}
declare -r KEYNAME=$(mktemp --tmpdir -u key-XXXXXX)
declare -r SSHKEYS=$(mktemp --tmpdir -u sshkeys-XXXXXX)
declare -r INSTANCE_NAME=$(mktemp -u test-XXXXXX | tr A-Z a-z)
declare -r MACHINE=${MACHINE:-n1-standard-1}
declare -r ZONE=${ZONE:-us-central1-f}
declare -r SUDO=${SUDO:-false}
# Standard arguments (applies only on script execution).
declare -ar SSH_ARGS=("-o" "ConnectTimeout=60" "--")
# This script is executed as a test rule, which will reset the value of HOME.
# Unfortunately, it is needed to load the gconfig credentials. We will reset
# HOME when we actually execute in the remote environment, defined below.
export HOME=$(eval echo ~$(whoami))
# Generate unique keys for this test.
[[ -f "${KEYNAME}" ]] || ssh-keygen -t rsa -N "" -f "${KEYNAME}" -C "${USERNAME}"
cat > "${SSHKEYS}" <<EOF
${USERNAME}:$(cat ${KEYNAME}.pub)
EOF
# Start a unique instance. This means that we first generate a unique set of ssh
# keys to ensure that only we have access to this instance. Note that we must
# constrain ourselves to Haswell or greater in order to have nested
# virtualization available.
gcloud compute instances create \
--min-cpu-platform "Intel Haswell" \
--preemptible \
--no-scopes \
--metadata block-project-ssh-keys=TRUE \
--metadata-from-file ssh-keys="${SSHKEYS}" \
--machine-type "${MACHINE}" \
--image "${IMAGE}" \
--zone "${ZONE}" \
"${INSTANCE_NAME}"
function cleanup {
gcloud compute instances delete --quiet --zone "${ZONE}" "${INSTANCE_NAME}"
}
trap cleanup EXIT
# Wait for the instance to become available (up to 5 minutes).
declare timeout=300
declare success=0
declare -r start=$(date +%s)
declare -r end=$((${start}+${timeout}))
while [[ "$(date +%s)" -lt "${end}" ]] && [[ "${success}" -lt 3 ]]; do
if gcloud compute ssh --ssh-key-file="${KEYNAME}" --zone "${ZONE}" "${USERNAME}"@"${INSTANCE_NAME}" -- true 2>/dev/null; then
success=$((${success}+1))
fi
done
if [[ "${success}" -eq "0" ]]; then
echo "connect timed out after ${timeout} seconds."
exit 1
fi
# Copy the local directory over.
tar czf - --dereference --exclude=.git . |
gcloud compute ssh \
--ssh-key-file="${KEYNAME}" \
--zone "${ZONE}" \
"${USERNAME}"@"${INSTANCE_NAME}" -- \
"${SSH_ARGS[@]}" \
tar xzf -
# Execute the command remotely.
for cmd; do
# Setup relevant environment.
#
# N.B. This is not a complete test environment, but is complete enough to
# provide rudimentary sharding and test output support.
declare -a PREFIX=( "env" )
if [[ -v TEST_SHARD_INDEX ]]; then
PREFIX+=( "TEST_SHARD_INDEX=${TEST_SHARD_INDEX}" )
fi
if [[ -v TEST_SHARD_STATUS_FILE ]]; then
SHARD_STATUS_FILE=$(mktemp -u test-shard-status-XXXXXX)
PREFIX+=( "TEST_SHARD_STATUS_FILE=/tmp/${SHARD_STATUS_FILE}" )
fi
if [[ -v TEST_TOTAL_SHARDS ]]; then
PREFIX+=( "TEST_TOTAL_SHARDS=${TEST_TOTAL_SHARDS}" )
fi
if [[ -v TEST_TMPDIR ]]; then
REMOTE_TMPDIR=$(mktemp -u test-XXXXXX)
PREFIX+=( "TEST_TMPDIR=/tmp/${REMOTE_TMPDIR}" )
# Create remotely.
gcloud compute ssh \
--ssh-key-file="${KEYNAME}" \
--zone "${ZONE}" \
"${USERNAME}"@"${INSTANCE_NAME}" -- \
"${SSH_ARGS[@]}" \
mkdir -p "/tmp/${REMOTE_TMPDIR}"
fi
if [[ -v XML_OUTPUT_FILE ]]; then
TEST_XML_OUTPUT=$(mktemp -u xml-output-XXXXXX)
PREFIX+=( "XML_OUTPUT_FILE=/tmp/${TEST_XML_OUTPUT}" )
fi
if [[ "${SUDO}" == "true" ]]; then
PREFIX+=( "sudo" "-E" )
fi
# Execute the command.
gcloud compute ssh \
--ssh-key-file="${KEYNAME}" \
--zone "${ZONE}" \
"${USERNAME}"@"${INSTANCE_NAME}" -- \
"${SSH_ARGS[@]}" \
"${PREFIX[@]}" "${cmd}"
# Collect relevant results.
if [[ -v TEST_SHARD_STATUS_FILE ]]; then
gcloud compute scp \
--ssh-key-file="${KEYNAME}" \
--zone "${ZONE}" \
"${USERNAME}"@"${INSTANCE_NAME}":/tmp/"${SHARD_STATUS_FILE}" \
"${TEST_SHARD_STATUS_FILE}" 2>/dev/null || true # Allowed to fail.
fi
if [[ -v XML_OUTPUT_FILE ]]; then
gcloud compute scp \
--ssh-key-file="${KEYNAME}" \
--zone "${ZONE}" \
"${USERNAME}"@"${INSTANCE_NAME}":/tmp/"${TEST_XML_OUTPUT}" \
"${XML_OUTPUT_FILE}" 2>/dev/null || true # Allowed to fail.
fi
# Clean up the temporary directory.
if [[ -v TEST_TMPDIR ]]; then
gcloud compute ssh \
--ssh-key-file="${KEYNAME}" \
--zone "${ZONE}" \
"${USERNAME}"@"${INSTANCE_NAME}" -- \
"${SSH_ARGS[@]}" \
rm -rf "/tmp/${REMOTE_TMPDIR}"
fi
done
tools/vm/test.cc
-27
View File
@@ -1,27 +0,0 @@
// Copyright 2020 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "gtest/gtest.h"
namespace {
TEST(Image, Sanity0) {
// Do nothing (in shard 0).
}
TEST(Image, Sanity1) {
// Do nothing (in shard 1).
}
} // namespace
tools/vm/ubuntu1604/10_core.sh
-43
View File
@@ -1,43 +0,0 @@
#!/bin/bash
# Copyright 2019 The gVisor Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -xeo pipefail
# Install all essential build tools.
while true; do
if (apt-get update && apt-get install -y \
make \
git-core \
build-essential \
linux-headers-$(uname -r) \
pkg-config); then
break
fi
result=$?
if [[ $result -ne 100 ]]; then
exit $result
fi
done
# Install a recent go toolchain.
if ! [[ -d /usr/local/go ]]; then
wget https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz
tar -xvf go1.13.5.linux-amd64.tar.gz
mv go /usr/local
fi
# Link the Go binary from /usr/bin; replacing anything there.
(cd /usr/bin && rm -f go && ln -fs /usr/local/go/bin/go go)
tools/vm/ubuntu1604/15_gcloud.sh
-50
View File
@@ -1,50 +0,0 @@
#!/bin/bash
# Copyright 2019 The gVisor Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -xeo pipefail
# Install all essential build tools.
while true; do
if (apt-get update && apt-get install -y \
apt-transport-https \
ca-certificates \
gnupg); then
break
fi
result=$?
if [[ $result -ne 100 ]]; then
exit $result
fi
done
# Add gcloud repositories.
echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | \
tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
# Add the appropriate key.
curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | \
apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
# Install the gcloud SDK.
while true; do
if (apt-get update && apt-get install -y google-cloud-sdk); then
break
fi
result=$?
if [[ $result -ne 100 ]]; then
exit $result
fi
done
tools/vm/ubuntu1604/20_bazel.sh
-38
View File
@@ -1,38 +0,0 @@
#!/bin/bash
# Copyright 2019 The gVisor Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -xeo pipefail
declare -r BAZEL_VERSION=2.0.0
# Install bazel dependencies.
while true; do
if (apt-get update && apt-get install -y \
openjdk-8-jdk-headless \
unzip); then
break
fi
result=$?
if [[ $result -ne 100 ]]; then
exit $result
fi
done
# Use the release installer.
curl -L -o bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh
chmod a+x bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh
./bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh
rm -f bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh
tools/vm/ubuntu1604/30_docker.sh
-64
View File
@@ -1,64 +0,0 @@
#!/bin/bash
# Copyright 2019 The gVisor Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Add dependencies.
while true; do
if (apt-get update && apt-get install -y \
apt-transport-https \
ca-certificates \
curl \
gnupg-agent \
software-properties-common); then
break
fi
result=$?
if [[ $result -ne 100 ]]; then
exit $result
fi
done
# Install the key.
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
# Add the repository.
add-apt-repository \
"deb [arch=amd64] https://download.docker.com/linux/ubuntu \
$(lsb_release -cs) \
stable"
# Install docker.
while true; do
if (apt-get update && apt-get install -y \
docker-ce \
docker-ce-cli \
containerd.io); then
break
fi
result=$?
if [[ $result -ne 100 ]]; then
exit $result
fi
done
# Enable experimental features, for cross-building aarch64 images.
# Enable Docker IPv6.
cat > /etc/docker/daemon.json <<EOF
{
"experimental": true,
"fixed-cidr-v6": "2001:db8:1::/64",
"ipv6": true
}
EOF
tools/vm/ubuntu1604/40_kokoro.sh
-72
View File
@@ -1,72 +0,0 @@
#!/bin/bash
# Copyright 2019 The gVisor Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -xeo pipefail
# Declare kokoro's required public keys.
declare -r ssh_public_keys=(
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDg7L/ZaEauETWrPklUTky3kvxqQfe2Ax/2CsSqhNIGNMnK/8d79CHlmY9+dE1FFQ/RzKNCaltgy7XcN/fCYiCZr5jm2ZtnLuGNOTzupMNhaYiPL419qmL+5rZXt4/dWTrsHbFRACxT8j51PcRMO5wgbL0Bg2XXimbx8kDFaurL2gqduQYqlu4lxWCaJqOL71WogcimeL63Nq/yeH5PJPWpqE4P9VUQSwAzBWFK/hLeds/AiP3MgVS65qHBnhq0JsHy8JQsqjZbG7Iidt/Ll0+gqzEbi62gDIcczG4KC0iOVzDDP/1BxDtt1lKeA23ll769Fcm3rJyoBMYxjvdw1TDx [email protected]"
"ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNgGK/hCdjmulHfRE3hp4rZs38NCR8yAh0eDsztxqGcuXnuSnL7jOlRrbcQpremJ84omD4eKrIpwJUs+YokMdv4= [email protected]"
)
# Install dependencies.
while true; do
if (apt-get update && apt-get install -y \
rsync \
coreutils \
python-psutil \
qemu-kvm \
python-pip \
python3-pip \
zip); then
break
fi
result=$?
if [[ $result -ne 100 ]]; then
exit $result
fi
done
# junitparser is used to merge junit xml files.
pip install --no-cache-dir junitparser
# We need a kbuilder user, which may already exist.
useradd -c "kbuilder user" -m -s /bin/bash kbuilder || true
# We need to provision appropriate keys.
mkdir -p ~kbuilder/.ssh
(IFS=$'\n'; echo "${ssh_public_keys[*]}") > ~kbuilder/.ssh/authorized_keys
chmod 0600 ~kbuilder/.ssh/authorized_keys
chown -R kbuilder ~kbuilder/.ssh
# Give passwordless sudo access.
cat > /etc/sudoers.d/kokoro <<EOF
kbuilder ALL=(ALL) NOPASSWD:ALL
EOF
# Ensure we can run Docker without sudo.
usermod -aG docker kbuilder
# Ensure that we can access kvm.
usermod -aG kvm kbuilder
# Ensure that /tmpfs exists and is writable by kokoro.
#
# Note that kokoro will typically attach a second disk (sdb) to the instance
# that is used for the /tmpfs volume. In the future we could setup an init
# script that formats and mounts this here; however, we don't expect our build
# artifacts to be that large.
mkdir -p /tmpfs && chmod 0777 /tmpfs && touch /tmpfs/READY
tools/vm/ubuntu1604/BUILD
-7
View File
@@ -1,7 +0,0 @@
package(licenses = ["notice"])
filegroup(
name = "ubuntu1604",
srcs = glob(["*.sh"]),
visibility = ["//:sandbox"],
)
tools/vm/ubuntu1804/BUILD
-7
View File
@@ -1,7 +0,0 @@
package(licenses = ["notice"])
alias(
name = "ubuntu1804",
actual = "//tools/vm/ubuntu1604",
visibility = ["//:sandbox"],
)
tools/vm/zone.sh
-17
View File
@@ -1,17 +0,0 @@
#!/bin/bash
# Copyright 2020 The gVisor Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
exec gcloud config get-value compute/zone