Merge pull request #14528 from EtiennePerot:zstd-release

PiperOrigin-RevId: 974154023

.github/workflows/release_mirror_lib.sh

@@ -34,6 +34,7 @@ declare -ra ARCHS=(x86_64 aarch64)
 # signed release metadata and the upgrade path that a release asset cannot.
 declare -ra RAW_FILES=(
   gvisor.tar.bz2
+  gvisor.tar.zstd
 )
 
 # raw_url <version> <arch> <filename>
@@ -47,7 +48,8 @@ raw_url() {
 # have to carry the architecture in their name.
 asset_name() {
   case "$1" in
-    *.tar.bz2) echo "${1%.tar.bz2}-$2.tar.bz2" ;;
-    *)         echo "$1-$2" ;;
+    *.tar.bz2)  echo "${1%.tar.bz2}-$2.tar.bz2" ;;
+    *.tar.zstd) echo "${1%.tar.zstd}-$2.tar.zstd" ;;
+    *)          echo "$1-$2" ;;
   esac
 }

Makefile

@@ -865,7 +865,8 @@ $(RELEASE_KEY):
 $(RELEASE_ARTIFACTS)/%:
 	@mkdir -p $@
 	@$(call copy,//debian:debian,$@)
-	@$(call copy,//debian:gvisor-release-tar,$@)
+	@$(call copy,//debian:gvisor-release-tar-bz2,$@)
+	@$(call copy,//debian:gvisor-release-tar-zstd,$@)
 
 release: $(RELEASE_KEY) $(RELEASE_ARTIFACTS)/$(ARCH)
 	@mkdir -p $(RELEASE_ROOT)
@@ -873,24 +874,25 @@ release: $(RELEASE_KEY) $(RELEASE_ARTIFACTS)/$(ARCH)
 .PHONY: release
 
 release-tarball: DESTINATION ?= .
-release-tarball: ## Builds an optimized release tarball (gvisor.tar.bz2) and copies it to $(DESTINATION). E.g. make release-tarball DESTINATION=bin/
+release-tarball: ## Builds optimized release tarballs (gvisor.tar.bz2, gvisor.tar.zstd) and copies them to $(DESTINATION). E.g. make release-tarball DESTINATION=bin/
 	@mkdir -p "$(DESTINATION)"
-	@$(call copy,-c opt //debian:gvisor-release-tar,$(DESTINATION))
+	@$(call copy,-c opt //debian:gvisor-release-tar-bz2,$(DESTINATION))
+	@$(call copy,-c opt //debian:gvisor-release-tar-zstd,$(DESTINATION))
 .PHONY: release-tarball
 
-staged-binaries-check: ## Verifies STAGED_BINARIES contains all files from the //debian:gvisor-release-tar fileset.
+staged-binaries-check: ## Verifies STAGED_BINARIES contains all files from the //debian:gvisor-release-tar-bz2 fileset.
 ifeq (,$(STAGED_BINARIES))
 	@echo "STAGED_BINARIES not set; nothing to check."
 else
 	@# T is exported so the nested `cp` inside the copy macro (a child process)
 	@# sees it; the rest of the recipe runs in this shell directly.
 	@export T=$$(mktemp -d --tmpdir staged-check.XXXXXX); \
-	$(call copy,//debian:gvisor-release-tar,$$T) && \
+	$(call copy,//debian:gvisor-release-tar-bz2,$$T) && \
 	tar -tjf "$$T/gvisor.tar.bz2" | sed 's#^\./##' | grep -v '/$$' | sort >"$$T/release.txt" && \
 	gcloud storage cat "$(STAGED_BINARIES)" | tar -tzf - | sed 's#^\./##' | grep -v '/$$' | sort >"$$T/staged.txt" && \
 	comm -23 "$$T/release.txt" "$$T/staged.txt" >"$$T/missing.txt" && \
 	test ! -s "$$T/missing.txt" \
-	  || { echo "ERROR: STAGED_BINARIES missing members from //debian:gvisor-release-tar:" >&2; cat "$$T/missing.txt" >&2; rm -rf "$$T"; exit 1; }; \
+	  || { echo "ERROR: STAGED_BINARIES missing members from //debian:gvisor-release-tar-bz2:" >&2; cat "$$T/missing.txt" >&2; rm -rf "$$T"; exit 1; }; \
 	rm -rf "$$T"
 endif
 .PHONY: staged-binaries-check

README.md

@@ -101,7 +101,7 @@ to get started:
 After setting up dependencies, using Bazel is similar to the Makefile:
 
 ```sh
-bazel build -c opt //debian:gvisor-release-tar
+bazel build -c opt //debian:gvisor-release-tar-bz2
 ```
 
 ### Testing

debian/BUILD

@@ -77,24 +77,42 @@ pkg_deb(
     ],
 )
 
+# Contents of the downloadable release tarballs.
+RELEASE_TAR_SRCS = [
+    "//runsc",
+    # containerd shim needs to be at the top level (not gvisor-bin/)
+    # because containerd requires it to be in $PATH.
+    "//shim:containerd-shim-runsc-v1",
+]
+
+RELEASE_TAR_DEPS = [
+    # gvisor-bin/ subdir contents:
+    ":gvisor-bin-tar",
+]
+
 # Downloadable release tarball containing all gVisor files.
 # This is not the Debian package, but it is in this BUILD file anyway
 # because it needs to contain the same set of files in practice, so
 # easier to keep in sync.
 pkg_tar(
-    name = "gvisor-release-tar",
-    srcs = [
-        "//runsc",
-        # containerd shim needs to be at the top level (not gvisor-bin/)
-        # because containerd requires it to be in $PATH.
-        "//shim:containerd-shim-runsc-v1",
-    ],
+    name = "gvisor-release-tar-bz2",
+    srcs = RELEASE_TAR_SRCS,
     extension = "tar.bz2",
     mode = "0755",
     package_file_name = "gvisor.tar.bz2",
     visibility = ["//visibility:public"],
-    deps = [
-        # gvisor-bin/ subdir contents:
-        ":gvisor-bin-tar",
-    ],
+    deps = RELEASE_TAR_DEPS,
+)
+
+# Same as :gvisor-release-tar-bz2 but zstd instead of bzip2
+# Smaller and much, much faster to extract.
+pkg_tar(
+    name = "gvisor-release-tar-zstd",
+    srcs = RELEASE_TAR_SRCS,
+    compressor = "//tools/zstd:compressor",
+    extension = "tar.zstd",
+    mode = "0755",
+    package_file_name = "gvisor.tar.zstd",
+    visibility = ["//visibility:public"],
+    deps = RELEASE_TAR_DEPS,
 )

g3doc/user_guide/install.md

@@ -49,17 +49,18 @@ To download and install the latest release manually follow these steps:
   set -e
   ARCH=$(uname -m)
   URL=https://storage.googleapis.com/gvisor/releases/release/latest/${ARCH}
-  wget ${URL}/gvisor.tar.bz2 ${URL}/gvisor.tar.bz2.sha512
-  sha512sum -c gvisor.tar.bz2.sha512
-  sudo tar -xjf gvisor.tar.bz2 -C /usr/local/bin
-  rm -f gvisor.tar.bz2 gvisor.tar.bz2.sha512
+  wget ${URL}/gvisor.tar.zstd ${URL}/gvisor.tar.zstd.sha512
+  sha512sum -c gvisor.tar.zstd.sha512
+  sudo tar --zstd -xf gvisor.tar.zstd -C /usr/local/bin
+  rm -f gvisor.tar.zstd gvisor.tar.zstd.sha512
 )
 ```
 
-The `gvisor.tar.bz2` tarball contains `runsc`, the `containerd-shim-runsc-v1`
+The `gvisor.tar.zstd` tarball contains `runsc`, the `containerd-shim-runsc-v1`
 containerd shim, and a `gvisor-bin/` directory with sidecar binaries that
 `runsc` executes at runtime. `runsc` looks for `gvisor-bin/` next to its own
-binary, so keep them together if you move them.
+binary, so keep them together if you move them. If you are running on a machine
+without `zstd` installed, you can use `gvisor.tar.bz2` instead.
 
 To install gVisor as a Docker runtime, run the following commands:
 

tools/make_release.sh

@@ -42,14 +42,15 @@ export DEBIAN_FRONTEND=noninteractive
 # install_raw installs raw artifacts.
 install_raw() {
   for binary in "${binaries[@]}"; do
-    local arch name
+    local arch file_info name
     # Copy the raw file & generate a sha512sum, sorted by architecture.
-    if echo "${binary}" | grep -qF .tar.bz2; then
-      # Determine arch from the `runsc` within the tarball:
-      arch=$(tar -xjOf "${binary}" runsc | file - | cut -d',' -f2 | awk '{print $NF}' | tr '-' '_')
-    else
-      arch=$(file "${binary}" | cut -d',' -f2 | awk '{print $NF}' | tr '-' '_')
-    fi
+    # For tarballs, determine arch from the `runsc` within the tarball.
+    case "${binary}" in
+      *.tar.bz2)  file_info=$(tar -xjOf "${binary}" runsc | file -) ;;
+      *.tar.zstd) file_info=$(tar --zstd -xOf "${binary}" runsc | file -) ;;
+      *)          file_info=$(file "${binary}") ;;
+    esac
+    arch=$(echo "${file_info}" | cut -d',' -f2 | awk '{print $NF}' | tr '-' '_')
     name=$(basename "${binary}")
     mkdir -p "${root}/$1/${arch}"
     cp -f "${binary}" "${root}/$1/${arch}"

tools/zstd/BUILD

@@ -0,0 +1,13 @@
+load("//tools:defs.bzl", "cc_binary")
+
+package(
+    default_applicable_licenses = ["//:license"],
+    licenses = ["notice"],
+)
+
+cc_binary(
+    name = "compressor",
+    srcs = ["compressor.c"],
+    visibility = ["//debian:__pkg__"],
+    deps = ["@llvm_zstd//:zstd"],
+)

tools/zstd/compressor.c

@@ -0,0 +1,100 @@
+// Copyright 2026 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This binary reads a tar stream on stdin and writes a zstd output to stdout.
+// It is intended to be used as `compressor` for a pkg_tar rule.
+// pkg_tar doesn't support zstd by default, and shelling out to the host's
+// zstd CLI results in non-deterministic output.
+//
+// The parameters that work best for the release are also non-standard (but
+// picked to still work with a default-implementation decoder).
+//
+// Specifically, we need:
+//   - Level 22 with long-distance matching. This is because we include many
+//     Go binaries (runsc, shim binary, metric server, sidecars) that share
+//     many of the same libraries and the Go runtime itself, and anything
+//     lower won't detect the overlap.
+//     Level 22 has 20% smaller size vs the previous one, it's a very
+//     measurable effect.
+//   - windowLog at 27 (i.e. 128MiB). This is the largest window that stock
+//     decoders (zstd -d, tar --zstd) can accept without a --long flag.
+//     Required to detect duplication within the `runsc` binary (which is
+//     large) together with the other Go binaries.
+//   - Single-threaded, so output bytes are reproducible and do not depend on
+//     the build machine.
+
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "lib/zstd.h"
+
+static void die(const char* msg) {
+  fprintf(stderr, "compressor: %s\n", msg);
+  exit(1);
+}
+
+static void set_param(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value) {
+  size_t ret = ZSTD_CCtx_setParameter(cctx, param, value);
+  if (ZSTD_isError(ret)) die(ZSTD_getErrorName(ret));
+}
+
+static void write_all(const char* buf, size_t len) {
+  while (len > 0) {
+    ssize_t n = write(STDOUT_FILENO, buf, len);
+    if (n < 0) {
+      if (errno == EINTR) continue;
+      die(strerror(errno));
+    }
+    buf += n;
+    len -= n;
+  }
+}
+
+int main(void) {
+  ZSTD_CCtx* cctx = ZSTD_createCCtx();
+  if (cctx == NULL) die("cannot create compression context");
+  set_param(cctx, ZSTD_c_compressionLevel, 22);
+  set_param(cctx, ZSTD_c_windowLog, 27);
+  set_param(cctx, ZSTD_c_enableLongDistanceMatching, 1);
+  set_param(cctx, ZSTD_c_checksumFlag, 1);
+
+  size_t const in_cap = ZSTD_CStreamInSize();
+  size_t const out_cap = ZSTD_CStreamOutSize();
+  char* in = malloc(in_cap);
+  char* out = malloc(out_cap);
+  if (in == NULL || out == NULL) die("cannot allocate buffers");
+
+  for (;;) {
+    ssize_t n = read(STDIN_FILENO, in, in_cap);
+    if (n < 0) {
+      if (errno == EINTR) continue;
+      die(strerror(errno));
+    }
+    ZSTD_EndDirective mode = n == 0 ? ZSTD_e_end : ZSTD_e_continue;
+    ZSTD_inBuffer input = {in, (size_t)n, 0};
+    for (;;) {
+      ZSTD_outBuffer output = {out, out_cap, 0};
+      size_t remaining = ZSTD_compressStream2(cctx, &output, &input, mode);
+      if (ZSTD_isError(remaining)) die(ZSTD_getErrorName(remaining));
+      write_all(out, output.pos);
+      if (mode == ZSTD_e_end ? remaining == 0 : input.pos == input.size) {
+        break;
+      }
+    }
+    if (n == 0) return 0;
+  }
+}