Detect if vvar is not mappable into the guest on ARM.

This adds a check if vvar cannot be mapped into the guest in a throw-away VM on
potentially problematic host kernels.

Linux commits 2a8dfab and 0c67288 make our mapping of vvar into the guest fail
as soon as we try to create it, because it is a VM_PFNMAP region.

Not mapping vvar into the guest severely hinders gVisor performance, because if
we don't map vvar then any time-related host syscall will always force a VM
exit, for example. However, given the alternative if/until a backport is merged
into the affected kernels is to just crash, we don't have much choice.

Note that this also revealed two errors:
- in KERNEL_ENTRY_FROM_EL1, which assumed R18 would not be relevant because
  Go never uses it -- this is no longer the case with the sentry faulting while
  in the VDSO code.
- in the interrupt stack size which was sized too small and thus corrupted
  errorCode and errorType variables when it overflowed. This is a problem with
  this patch specifically, because we now generate a data abort exception every
  time we use vvar, which exposes the bug more frequently.

  This also led to a closer inspection of the EXCEPTION_EL1 macro, which
  revealed a relatively harmless bug of vector not being saved properly.

PiperOrigin-RevId: 968804355

pkg/ring0/defs_arm64.go

@@ -36,11 +36,8 @@ const (
 type KernelArchState struct {
 }
 
-// CPUArchState contains CPU-specific arch state.
-type CPUArchState struct {
-	// stack is the stack used for interrupts on this CPU.
-	stack [128]byte
-
+// CPUArchRegisters contains non-stack CPU architecture state.
+type CPUArchRegisters struct {
 	// errorCode is the error code from the last exception.
 	errorCode uintptr
 
@@ -78,6 +75,19 @@ type CPUArchState struct {
 	appASID uintptr
 }
 
+// CPUArchState contains CPU-specific arch state.
+type CPUArchState struct {
+	// stack is the stack used for interrupts on this CPU.
+	//
+	// The EL1 trampolines (HaltEl1SvcAndResume, HaltEl1ExceptionAndResume)
+	// run Go call chains on this stack. The deepest chain
+	// (kernelException -> vCPU.KernelException -> Halt) needs ~136 bytes
+	// plus 16 bytes for HaltEl1ExceptionAndResume itself to stage args.
+	stack [176]byte
+
+	CPUArchRegisters
+}
+
 // ErrorCode returns the last error code.
 //
 // The returned boolean indicates whether the error code corresponds to the

pkg/ring0/entry_arm64.s

@@ -16,20 +16,20 @@
 #include "textflag.h"
 
 #define CPU_SELF             0   // +checkoffset . CPU.self
-#define CPU_REGISTERS        224 // +checkoffset . CPU.registers
+#define CPU_REGISTERS        272 // +checkoffset . CPU.registers
 #define CPU_ARCH_STATE       16  // +checkoffset . CPU.CPUArchState
-#define CPU_STACK_BOTTOM     CPU_ARCH_STATE+0     // +checkoffset . CPUArchState.stack
-#define CPU_STACK_TOP        CPU_STACK_BOTTOM+128 // +checksize . CPUArchState.stack
-#define CPU_ERROR_CODE       CPU_ARCH_STATE+128   // +checkoffset . CPUArchState.errorCode
-#define CPU_ERROR_TYPE       CPU_ARCH_STATE+136   // +checkoffset . CPUArchState.errorType
-#define CPU_FAULT_ADDR       CPU_ARCH_STATE+144   // +checkoffset . CPUArchState.faultAddr
-#define CPU_FPSTATE_EL0      CPU_ARCH_STATE+152   // +checkoffset . CPUArchState.el0Fp
-#define CPU_TTBR0_KVM        CPU_ARCH_STATE+160   // +checkoffset . CPUArchState.ttbr0Kvm
-#define CPU_TTBR0_APP        CPU_ARCH_STATE+168   // +checkoffset . CPUArchState.ttbr0App
-#define CPU_VECTOR_CODE      CPU_ARCH_STATE+176   // +checkoffset . CPUArchState.vecCode
-#define CPU_APP_ADDR         CPU_ARCH_STATE+184   // +checkoffset . CPUArchState.appAddr
-#define CPU_LAZY_VFP         CPU_ARCH_STATE+192   // +checkoffset . CPUArchState.lazyVFP
-#define CPU_APP_ASID         CPU_ARCH_STATE+200   // +checkoffset . CPUArchState.appASID
+#define CPU_STACK_BOTTOM     CPU_ARCH_STATE+0                   // +checkoffset . CPUArchState.stack
+#define CPU_STACK_TOP        CPU_STACK_BOTTOM+176               // +checksize . CPUArchState.stack
+#define CPU_ERROR_CODE       CPU_STACK_TOP+0                    // +checkoffset . CPUArchRegisters.errorCode
+#define CPU_ERROR_TYPE       CPU_STACK_TOP+8                    // +checkoffset . CPUArchRegisters.errorType
+#define CPU_FAULT_ADDR       CPU_STACK_TOP+16                   // +checkoffset . CPUArchRegisters.faultAddr
+#define CPU_FPSTATE_EL0      CPU_STACK_TOP+24                   // +checkoffset . CPUArchRegisters.el0Fp
+#define CPU_TTBR0_KVM        CPU_STACK_TOP+32                   // +checkoffset . CPUArchRegisters.ttbr0Kvm
+#define CPU_TTBR0_APP        CPU_STACK_TOP+40                   // +checkoffset . CPUArchRegisters.ttbr0App
+#define CPU_VECTOR_CODE      CPU_STACK_TOP+48                   // +checkoffset . CPUArchRegisters.vecCode
+#define CPU_APP_ADDR         CPU_STACK_TOP+56                   // +checkoffset . CPUArchRegisters.appAddr
+#define CPU_LAZY_VFP         CPU_STACK_TOP+64                   // +checkoffset . CPUArchRegisters.lazyVFP
+#define CPU_APP_ASID         CPU_STACK_TOP+72                   // +checkoffset . CPUArchRegisters.appASID
 
 // Bits.
 #define _KERNEL_FLAGS 965 // +checkconst . KernelFlagsSet
@@ -397,10 +397,18 @@ TEXT ·FPSIMDEnableTrap(SB),NOSPLIT,$0
 	MOVD R3, PTRACE_SP(R20);
 
 // KERNEL_ENTRY_FROM_EL1 is the entry code of the vcpu from el1 to el1.
+//
+// Match KERNEL_ENTRY_FROM_EL0: r18 must be saved as well because EL1 exceptions
+// can also interrupt the host vdso executing in guest mode, and the vdso C code
+// that may freely use r18 as a temporary.
 #define KERNEL_ENTRY_FROM_EL1 \
+	SUB $16, RSP, RSP; \		// save r18, r19 into the stack.
+	STP (RSV_REG, RSV_REG_APP), 16*0(RSP); \
 	WORD $0xd538d092; \   //MRS   TPIDR_EL1, R18
-	REGISTERS_SAVE(RSV_REG, CPU_REGISTERS); \	// Save sentry context.
-	MOVD RSV_REG_APP, CPU_REGISTERS+PTRACE_R19(RSV_REG); \
+	REGISTERS_SAVE(RSV_REG, CPU_REGISTERS); \	// save sentry context (except r18, r19).
+	LDP 16*0(RSP), (R4, R5); \	// load the original r18, r19.
+	ADD $16, RSP, RSP; \
+	STP (R4, R5), CPU_REGISTERS+PTRACE_R18(RSV_REG); \	// save the original r18, r19.
 	MRS TPIDR_EL0, R4; \
 	MOVD R4, CPU_REGISTERS+PTRACE_TLS(RSV_REG); \
 	WORD $0xd5384004; \    //    MRS SPSR_EL1, R4
@@ -426,6 +434,7 @@ TEXT ·FPSIMDEnableTrap(SB),NOSPLIT,$0
 
 // EXCEPTION_EL1 is a common el1 exception handler function.
 #define EXCEPTION_EL1(vector) \
+	SUB $16, RSP, RSP; \	// Reserve for HaltEl1ExceptionAndResume
 	MOVD $vector, R3; \
 	MOVD R3, 8(RSP); \
 	B ·HaltEl1ExceptionAndResume(SB);
@@ -483,10 +492,10 @@ TEXT ·HaltEl1SvcAndResume(SB),NOSPLIT,$0
 // HaltEl1ExceptionAndResume calls Hooks.KernelException and resume.
 TEXT ·HaltEl1ExceptionAndResume(SB),NOSPLIT,$0
 	WORD $0xd538d092            // MRS TPIDR_EL1, R18
-	MOVD CPU_SELF(RSV_REG), R3  // Load vCPU.
-	MOVD R3, 8(RSP)             // First argument (vCPU).
 	MOVD vector+0(FP), R3
 	MOVD R3, 16(RSP)            // Second argument (vector).
+	MOVD CPU_SELF(RSV_REG), R3  // Load vCPU.
+	MOVD R3, 8(RSP)             // First argument (vCPU).
 	CALL ·kernelException(SB)   // Call the trampoline.
 	B ·kernelExitToEl1(SB)      // Resume.
 
@@ -585,6 +594,7 @@ TEXT ·kernelExitToEl1(SB),NOSPLIT,$0
 	MRS TPIDR_EL1, RSV_REG
 
 	MOVD CPU_REGISTERS+PTRACE_R19(RSV_REG), RSV_REG_APP
+	MOVD CPU_REGISTERS+PTRACE_R18(RSV_REG), RSV_REG
 
 	ERET()
 

pkg/sentry/platform/kvm/kvm_arm64.go

@@ -23,6 +23,7 @@ import (
 	"golang.org/x/sys/unix"
 	"gvisor.dev/gvisor/pkg/abi/linux"
 	"gvisor.dev/gvisor/pkg/hostsyscall"
+	"gvisor.dev/gvisor/pkg/log"
 	"gvisor.dev/gvisor/pkg/ring0"
 	"gvisor.dev/gvisor/pkg/sentry/arch"
 )
@@ -80,6 +81,12 @@ func updateGlobalOnce(fd int) error {
 
 	ring0.Init()
 	initFaultBlocks()
+
+	excludeVvarRegion = vvarMemslotUnusable(fd)
+	if excludeVvarRegion {
+		// Not a warning to make tests pass that match on not getting any warnings.
+		log.Infof("[vvar] region will not be mapped into the KVM guest! Performance will suffer!")
+	}
 	physicalInit()
 
 	// Explicitly configure address space for 48-bit VA.

pkg/sentry/platform/kvm/kvm_arm64_unsafe.go

@@ -19,9 +19,12 @@ package kvm
 
 import (
 	"fmt"
+	"unsafe"
 
 	"golang.org/x/sys/unix"
+	"gvisor.dev/gvisor/pkg/hostos"
 	"gvisor.dev/gvisor/pkg/hostsyscall"
+	"gvisor.dev/gvisor/pkg/log"
 )
 
 var (
@@ -42,3 +45,43 @@ func updateSystemValues(fd int) error {
 	// Success.
 	return nil
 }
+
+// vvarMemslotUnusable checks if mapping vvar into a memslot is liable to being
+// blocked by the host. This can happen due to kernel commits 2a8dfab and
+// 0c67288.
+func vvarMemslotUnusable(kvmFd int) bool {
+	version, err := hostos.KernelVersion()
+	// Problematic kernel version range:
+	if err == nil && (version.LessThan(6, 17) || version.AtLeast(7, 1)) {
+		return false
+	}
+
+	vvar, err := findRegionStartingWith("[vvar")
+	if err != nil {
+		// Very unexpected...
+		log.Warningf("%v", err)
+		// Let's just try our luck
+		// or: we didn't find vvar once, we can't find it a second time.
+		return false
+	}
+	if vvar.length == 0 {
+		return false
+	}
+
+	vmFd, errno := hostsyscall.RawSyscall(unix.SYS_IOCTL, uintptr(kvmFd), KVM_CREATE_VM, 0)
+	if errno != 0 {
+		// We will re-fail on the actual VM creation.
+		return false
+	}
+	defer hostsyscall.RawSyscall(unix.SYS_CLOSE, vmFd, 0, 0)
+
+	userRegion := userMemoryRegion{
+		slot:          0,
+		flags:         _KVM_MEM_READONLY,
+		guestPhysAddr: 0,
+		memorySize:    uint64(vvar.length),
+		userspaceAddr: uint64(uintptr(vvar.virtual)),
+	}
+	_, errno = hostsyscall.RawSyscall(unix.SYS_IOCTL, vmFd, KVM_SET_USER_MEMORY_REGION, uintptr(unsafe.Pointer(&userRegion)))
+	return errno != 0
+}

pkg/sentry/platform/kvm/machine.go

@@ -396,9 +396,9 @@ func newMachine(vm int, config *Config) (*machine, error) {
 	// called while we hold the slot spinlock.
 	disableAsyncPreemption()
 
-	applyVirtualRegions(func(vr virtualRegion) {
+	applyVirtualRegions(func(vr virtualRegion) bool {
 		if excludeVirtualRegion(vr) {
-			return // skip region.
+			return false // skip region.
 		}
 		// Take into account that the stack can grow down.
 		if vr.filename == "[stack]" {
@@ -407,7 +407,7 @@ func newMachine(vm int, config *Config) (*machine, error) {
 		}
 
 		mapRegion(vr, 0)
-
+		return false
 	})
 	if mapEntireAddressSpace {
 		for _, r := range physicalRegions {

pkg/sentry/platform/kvm/machine_amd64.go

@@ -525,9 +525,9 @@ func (m *machine) mapUpperHalfRegion(
 func (m *machine) mapUpperHalf(pageTable *pagetables.PageTables) {
 	// Map all the executable regions so that all the entry functions
 	// are mapped in the upper half.
-	if err := applyVirtualRegions(func(vr virtualRegion) {
+	if err := applyVirtualRegions(func(vr virtualRegion) bool {
 		if excludeVirtualRegion(vr) || vr.filename == "[vsyscall]" {
-			return
+			return false
 		}
 
 		if vr.accessType.Execute {
@@ -535,6 +535,7 @@ func (m *machine) mapUpperHalf(pageTable *pagetables.PageTables) {
 			m.mapUpperHalfRegion(pageTable, r.virtual, r.length,
 				pagetables.MapOpts{AccessType: hostarch.Execute, Global: true})
 		}
+		return false
 	}); err != nil {
 		panic(fmt.Sprintf("error parsing /proc/self/maps: %v", err))
 	}

pkg/sentry/platform/kvm/machine_arm64.go

@@ -73,15 +73,16 @@ func (m *machine) mapUpperHalf(pageTable *pagetables.PageTables) {
 // physical regions form them.
 func archPhysicalRegions(physicalRegions []physicalRegion) []physicalRegion {
 	rdRegions := []virtualRegion{}
-	if err := applyVirtualRegions(func(vr virtualRegion) {
+	if err := applyVirtualRegions(func(vr virtualRegion) bool {
 		if excludeVirtualRegion(vr) {
-			return // skip region.
+			return false // skip region.
 		}
 		// Skip PROT_NONE mappings. Go-runtime uses them as place
 		// holders for future read-write mappings.
 		if !vr.accessType.Write && vr.accessType.Read {
 			rdRegions = append(rdRegions, vr)
 		}
+		return false
 	}); err != nil {
 		panic(fmt.Sprintf("error parsing /proc/self/maps: %v", err))
 	}

pkg/sentry/platform/kvm/physical_map.go

@@ -32,7 +32,8 @@ type region struct {
 
 type specialVirtualRegion struct {
 	region
-	mmio bool // if false, region is excluded
+	filename string
+	mmio     bool // if false, region is excluded
 }
 
 type physicalRegion struct {
@@ -67,22 +68,25 @@ func fillAddressSpace() (specialRegions []specialVirtualRegion) {
 	pSize -= reservedMemory
 
 	// Add specifically excluded regions; see excludeVirtualRegion.
-	if err := applyVirtualRegions(func(vr virtualRegion) {
+	if err := applyVirtualRegions(func(vr virtualRegion) bool {
 		if excludeVirtualRegion(vr) {
 			specialRegions = append(specialRegions, specialVirtualRegion{
-				region: vr.region,
+				region:   vr.region,
+				filename: vr.filename,
 			})
 			vSize -= vr.length
 			log.Infof("excluded: virtual [%x,%x)", vr.virtual, vr.virtual+vr.length)
 		} else if mmioVirtualRegion(vr) {
 			specialRegions = append(specialRegions, specialVirtualRegion{
-				region: vr.region,
-				mmio:   true,
+				region:   vr.region,
+				filename: vr.filename,
+				mmio:     true,
 			})
 			log.Infof("mmio: virtual [%x,%x)", vr.virtual, vr.virtual+vr.length)
 		} else {
 			archSpecialRegion(vr)
 		}
+		return false // never stop
 	}); err != nil {
 		panic(fmt.Sprintf("error parsing /proc/self/maps: %v", err))
 	}
@@ -172,6 +176,7 @@ func fillAddressSpace() (specialRegions []specialVirtualRegion) {
 					virtual: addr,
 					length:  current,
 				},
+				filename: "[anon]",
 			})
 			// See comment above.
 			if filled != required {
@@ -181,12 +186,13 @@ func fillAddressSpace() (specialRegions []specialVirtualRegion) {
 	}
 	if current == 0 {
 		log.Warningf("Filling address space failed (VirtualAddressBits=%d PhysicalAddressBits=%d vSize=%#x pSize=%#x faultBlockSize=%#x required=%#x filled=%#x); last mmap errno: %v; VMAs:", ring0.VirtualAddressBits, ring0.PhysicalAddressBits, vSize, pSize, faultBlockSize, required, filled, lastMmapErrno)
-		err := applyVirtualRegions(func(vr virtualRegion) {
+		err := applyVirtualRegions(func(vr virtualRegion) bool {
 			ps := "p"
 			if vr.shared {
 				ps = "s"
 			}
 			log.Warningf("%x-%x %v%s %08x %s", vr.region.virtual, vr.region.virtual+vr.region.length, vr.accessType, ps, vr.offset, vr.filename)
+			return false
 		})
 		if err != nil {
 			log.Warningf("Failed to get all VMAs: %v", err)
@@ -197,7 +203,7 @@ func fillAddressSpace() (specialRegions []specialVirtualRegion) {
 		return specialRegions[i].virtual < specialRegions[j].virtual
 	})
 	for _, r := range specialRegions {
-		log.Infof("special region: virtual [%x,%x)", r.virtual, r.virtual+r.length)
+		log.Infof("special region %s: virtual [%x,%x)", r.filename, r.virtual, r.virtual+r.length)
 	}
 	return specialRegions
 }

pkg/sentry/platform/kvm/physical_map_amd64.go

@@ -77,7 +77,8 @@ func archSpecialRegions(vSize uintptr) (uintptr, []specialVirtualRegion) {
 				length:  vr.virtual - start,
 			}
 			specialRegions = append(specialRegions, specialVirtualRegion{
-				region: r,
+				region:   r,
+				filename: "[extended]",
 			})
 			vSize -= r.length
 			log.Infof("excluded: virtual [%x,%x)", r.virtual, r.virtual+r.length)

pkg/sentry/platform/kvm/virtual_map.go

@@ -21,6 +21,7 @@ import (
 	"os"
 	"regexp"
 	"strconv"
+	"strings"
 
 	"gvisor.dev/gvisor/pkg/hostarch"
 )
@@ -36,13 +37,15 @@ type virtualRegion struct {
 // mapsLine matches a single line from /proc/PID/maps.
 var mapsLine = regexp.MustCompile(`([0-9a-f]+)-([0-9a-f]+) ([r-][w-][x-][sp]) ([0-9a-f]+) [0-9a-f]{2,3}:[0-9a-f]{2,} [0-9]+\s+(.*)`)
 
+var excludeVvarRegion bool
+
 // excludeVirtualRegion returns true if these regions should be excluded from
 // the physical map. Virtual regions need to be excluded if get_user_pages will
 // fail on those addresses, preventing KVM from satisfying EPT faults.
 //
 // This is called by the physical map functions, not applyVirtualRegions.
 func excludeVirtualRegion(r virtualRegion) bool {
-	return false
+	return excludeVvarRegion && (strings.HasPrefix(r.filename, "[vvar") || strings.HasPrefix(r.filename, "[vdso"))
 }
 
 // mmioVirtualRegion returns true if r should be included in the physical map,
@@ -56,8 +59,11 @@ func mmioVirtualRegion(r virtualRegion) bool {
 
 // applyVirtualRegions parses the process maps file.
 //
+// Takes a function that's called on each virtualRegion. If function returns
+// true, stops iteration.
+//
 // Unlike mappedRegions, these are not consistent over time.
-func applyVirtualRegions(fn func(vr virtualRegion)) error {
+func applyVirtualRegions(fn func(vr virtualRegion) bool) error {
 	// Open /proc/self/maps.
 	f, err := os.Open("/proc/self/maps")
 	if err != nil {
@@ -91,7 +97,7 @@ func applyVirtualRegions(fn func(vr virtualRegion)) error {
 			if err != nil {
 				return fmt.Errorf("bad offset: %v", string(b))
 			}
-			fn(virtualRegion{
+			if stop := fn(virtualRegion{
 				region: region{
 					virtual: uintptr(start),
 					length:  uintptr(end - start),
@@ -104,7 +110,9 @@ func applyVirtualRegions(fn func(vr virtualRegion)) error {
 				shared:   shared,
 				offset:   uintptr(offset),
 				filename: string(m[5]),
-			})
+			}); stop {
+				break
+			}
 		}
 		if err != nil && err == io.EOF {
 			break
@@ -115,3 +123,23 @@ func applyVirtualRegions(fn func(vr virtualRegion)) error {
 
 	return nil
 }
+
+func findRegionStartingWith(name string) (region, error) {
+	var r region
+	var found bool
+	err := applyVirtualRegions(func(vr virtualRegion) bool {
+		if strings.HasPrefix(vr.filename, name) {
+			r = vr.region
+			found = true
+			return true
+		}
+		return false
+	})
+	if err != nil {
+		return region{}, err
+	}
+	if !found {
+		return region{}, fmt.Errorf("region %s not found", name)
+	}
+	return r, nil
+}

pkg/sentry/platform/kvm/virtual_map_test.go

@@ -26,13 +26,15 @@ type checker struct {
 	accessType hostarch.AccessType
 }
 
-func (c *checker) Containing(addr uintptr) func(virtualRegion) {
+func (c *checker) Containing(addr uintptr) func(virtualRegion) bool {
 	c.ok = false // Reset for below calls.
-	return func(vr virtualRegion) {
+	return func(vr virtualRegion) bool {
 		if vr.virtual <= addr && addr < vr.virtual+vr.length {
 			c.ok = true
 			c.accessType = vr.accessType
+			return true
 		}
+		return false
 	}
 }