visor/pkg/sentry/platform/kvm/machine_arm64_unsafe.go

// Copyright 2019 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//go:build arm64
// +build arm64

package kvm

import (
	"fmt"
	"reflect"
	"unsafe"

	"golang.org/x/sys/unix"
	"gvisor.dev/gvisor/pkg/abi/linux"
	"gvisor.dev/gvisor/pkg/hostarch"
	"gvisor.dev/gvisor/pkg/hostsyscall"
	"gvisor.dev/gvisor/pkg/ring0"
	"gvisor.dev/gvisor/pkg/ring0/pagetables"
	"gvisor.dev/gvisor/pkg/sentry/platform"
	ktime "gvisor.dev/gvisor/pkg/sentry/time"
)

type kvmVcpuInit struct {
	target   uint32
	features [7]uint32
}

var vcpuInit kvmVcpuInit

// ptrauthKeyRegs is the set of pointer authentication key registers.
var ptrauthKeyRegs = [...]uint64{
	_KVM_ARM64_REGS_APIAKEYLO_EL1,
	_KVM_ARM64_REGS_APIAKEYHI_EL1,
	_KVM_ARM64_REGS_APIBKEYLO_EL1,
	_KVM_ARM64_REGS_APIBKEYHI_EL1,
	_KVM_ARM64_REGS_APDAKEYLO_EL1,
	_KVM_ARM64_REGS_APDAKEYHI_EL1,
	_KVM_ARM64_REGS_APDBKEYLO_EL1,
	_KVM_ARM64_REGS_APDBKEYHI_EL1,
	_KVM_ARM64_REGS_APGAKEYLO_EL1,
	_KVM_ARM64_REGS_APGAKEYHI_EL1,
}

// initArchState initializes architecture-specific state.
func (m *machine) initArchState() error {
	if errno := hostsyscall.RawSyscallErrno(
		unix.SYS_IOCTL,
		uintptr(m.fd),
		_KVM_ARM_PREFERRED_TARGET,
		uintptr(unsafe.Pointer(&vcpuInit))); errno != 0 {
		panic(fmt.Sprintf("error setting KVM_ARM_PREFERRED_TARGET failed: %v", errno))
	}

	// Initialize all vCPUs on ARM64 (we also do this on x86_64, but for
	// ARM64 it is especially important as it has a different KVM timer
	// mechanism).
	// If we create vCPU dynamically on ARM64, the timer for vCPU would mess
	// up for a short time.
	// For more detail, please refer to https://github.com/google/gvisor/issues/5739
	m.mu.Lock()
	defer m.mu.Unlock()
	for i := 0; i < m.maxVCPUs; i++ {
		if _, err := m.createVCPU(i); err != nil {
			return err
		}
	}
	return nil
}

// initArchState initializes architecture-specific state.
func (c *vCPU) initArchState() error {
	var (
		reg     kvmOneReg
		data    uint64
		regGet  kvmOneReg
		dataGet uint64
	)

	reg.addr = uint64(reflect.ValueOf(&data).Pointer())
	regGet.addr = uint64(reflect.ValueOf(&dataGet).Pointer())

	vcpuInit.features[0] |= (1 << _KVM_ARM_VCPU_PSCI_0_2)
	if hasPtrauth {
		// Enable pointer authentication in the guest. Without this,
		// KVM injects an undefined instruction exception for every
		// non-hint PAC instruction (e.g. PACIA, RETAA, PACGA), which
		// are UNDEFINED without FEAT_PAuth; applications compiled for
		// armv8.3+ execute them unconditionally and would receive
		// spurious SIGILLs. KVM requires both features to be set
		// together.
		vcpuInit.features[0] |= (1 << _KVM_ARM_VCPU_PTRAUTH_ADDRESS) |
			(1 << _KVM_ARM_VCPU_PTRAUTH_GENERIC)
	}
	if errno := hostsyscall.RawSyscallErrno(
		unix.SYS_IOCTL,
		uintptr(c.fd),
		_KVM_ARM_VCPU_INIT,
		uintptr(unsafe.Pointer(&vcpuInit))); errno != 0 {
		panic(fmt.Sprintf("error setting KVM_ARM_VCPU_INIT failed: %v", errno))
	}

	if hasPtrauth {
		// Zero all pointer authentication keys.
		//
		// _SCTLR_EL1_DEFAULT keeps EnIA/EnIB/EnDA/EnDB clear, so the
		// address authentication instructions execute as no-ops at both
		// EL1 and EL0, matching a host with pointer authentication
		// disabled (see disableHostPAC()). PACGA, however, is not
		// gated by SCTLR_EL1 and always mixes in APGAKey_EL1, and KVM
		// resets the key registers to an unspecified value. Zeroed
		// keys keep PACGA results consistent across vCPUs, which is
		// required because tasks are not pinned to vCPUs.
		for _, id := range ptrauthKeyRegs {
			data = 0
			reg.id = id
			if err := c.setOneRegister(&reg); err != nil {
				return err
			}
		}
	}

	// tcr_el1
	data = _TCR_TXSZ_VA48 | _TCR_CACHE_FLAGS | _TCR_SHARED | _TCR_TG_FLAGS |
		_TCR_ASID16 | _TCR_IPS_40BITS | _TCR_TBI0
	reg.id = _KVM_ARM64_REGS_TCR_EL1
	if err := c.setOneRegister(&reg); err != nil {
		return err
	}

	// mair_el1
	if hostarch.NumMemoryTypes != 3 {
		panic("additional memory types must be configured in MAIR")
	}
	data = (_MT_ATTR_NORMAL << (hostarch.MemoryTypeWriteBack * 8)) |
		(_MT_ATTR_NORMAL_NC << (hostarch.MemoryTypeWriteCombine * 8)) |
		(_MT_ATTR_DEVICE_nGnRnE << (hostarch.MemoryTypeUncached * 8))
	reg.id = _KVM_ARM64_REGS_MAIR_EL1
	if err := c.setOneRegister(&reg); err != nil {
		return err
	}

	// ttbr0_el1
	data = c.machine.kernel.PageTables.TTBR0_EL1(false, 0)

	reg.id = _KVM_ARM64_REGS_TTBR0_EL1
	if err := c.setOneRegister(&reg); err != nil {
		return err
	}

	c.SetTtbr0Kvm(uintptr(data))

	// ttbr1_el1
	data = c.machine.kernel.PageTables.TTBR1_EL1(false, 0)

	reg.id = _KVM_ARM64_REGS_TTBR1_EL1
	if err := c.setOneRegister(&reg); err != nil {
		return err
	}

	// cntkctl_el1
	data = _CNTKCTL_EL1_DEFAULT
	reg.id = _KVM_ARM64_REGS_CNTKCTL_EL1
	if err := c.setOneRegister(&reg); err != nil {
		return err
	}

	// cpacr_el1
	data = 0
	reg.id = _KVM_ARM64_REGS_CPACR_EL1
	if err := c.setOneRegister(&reg); err != nil {
		return err
	}

	// sctlr_el1
	data = _SCTLR_EL1_DEFAULT
	reg.id = _KVM_ARM64_REGS_SCTLR_EL1
	if err := c.setOneRegister(&reg); err != nil {
		return err
	}

	// tpidr_el1
	reg.id = _KVM_ARM64_REGS_TPIDR_EL1
	data = uint64(reflect.ValueOf(&c.CPU).Pointer() | ring0.KernelStartAddress)
	if err := c.setOneRegister(&reg); err != nil {
		return err
	}

	// sp_el1
	data = c.CPU.StackTop()
	reg.id = _KVM_ARM64_REGS_SP_EL1
	if err := c.setOneRegister(&reg); err != nil {
		return err
	}

	// pc
	reg.id = _KVM_ARM64_REGS_PC
	data = uint64(ring0.AddrOfStart())
	if err := c.setOneRegister(&reg); err != nil {
		return err
	}

	// vbar_el1
	reg.id = _KVM_ARM64_REGS_VBAR_EL1
	vectorLocation := ring0.AddrOfVectors()
	data = uint64(ring0.KernelStartAddress | vectorLocation)
	if err := c.setOneRegister(&reg); err != nil {
		return err
	}

	// Use the address of the exception vector table as
	// the MMIO address base.
	vectorLocationPhys, _, _ := translateToPhysical(vectorLocation)
	arm64HypercallMMIOBase = vectorLocationPhys

	// Initialize the PCID database.
	if hasGuestPCID {
		// Note that NewPCIDs may return a nil table here, in which
		// case we simply don't use PCID support (see below). In
		// practice, this should not happen, however.
		c.PCIDs = pagetables.NewPCIDs(fixedKernelPCID+1, poolPCIDs)
	}

	return c.setSystemTime()
}

// setTSC sets the counter Virtual Offset.
func (c *vCPU) setTSC(value uint64) error {
	var (
		reg  kvmOneReg
		data uint64
	)

	reg.addr = uint64(reflect.ValueOf(&data).Pointer())
	reg.id = _KVM_ARM64_REGS_TIMER_CNT
	data = uint64(value)

	if err := c.setOneRegister(&reg); err != nil {
		return err
	}

	return nil
}

// getTSC gets the counter Physical Counter minus Virtual Offset.
func (c *vCPU) getTSC() error {
	var (
		reg  kvmOneReg
		data uint64
	)

	reg.addr = uint64(reflect.ValueOf(&data).Pointer())
	reg.id = _KVM_ARM64_REGS_TIMER_CNT

	if errno := c.getOneRegister(&reg); errno != 0 {
		return fmt.Errorf("error getting KVM_ARM64_REGS_TIMER_CNT: %w", errno)
	}

	return nil
}

// setSystemTime sets the vCPU to the system time.
func (c *vCPU) setSystemTime() error {
	const minIterations = 10
	minimum := uint64(0)
	for iter := 0; ; iter++ {
		// Use get the TSC to an estimate of where it will be
		// on the host during a "fast" system call iteration.
		// replace getTSC to another setOneRegister syscall can get more accurate value?
		start := uint64(ktime.Rdtsc())
		if err := c.getTSC(); err != nil {
			return err
		}
		// See if this is our new minimum call time. Note that this
		// serves two functions: one, we make sure that we are
		// accurately predicting the offset we need to set. Second, we
		// don't want to do the final set on a slow call, which could
		// produce a really bad result.
		end := uint64(ktime.Rdtsc())
		if end < start {
			continue // Totally bogus: unstable TSC?
		}
		current := end - start
		if current < minimum || iter == 0 {
			minimum = current // Set our new minimum.
		}
		// Is this past minIterations and within ~10% of minimum?
		upperThreshold := (((minimum << 3) + minimum) >> 3)
		if iter >= minIterations && (current <= upperThreshold || minimum < 50) {
			// Try to set the TSC
			if err := c.setTSC(end + (minimum / 2)); err != nil {
				return err
			}
			return nil
		}
	}
}

//go:nosplit
func (c *vCPU) loadSegments(tid uint64) {
	// Disable host pointer authentication on this thread before any
	// bluepill entry. See disableHostPAC() for the rationale.
	disableHostPAC()

	// TODO(gvisor.dev/issue/1238):  TLS is not supported.
	// Get TLS from tpidr_el0.
	c.tid.Store(tid)
}

// checkPAC is a quick check to see if the host CPU supports pointer
// authentication to avoid having to do a host-syscall if it's unnecessary.
//
//go:nosplit
func checkPAC() bool

// addressPACKeys is the bitmask of every ARM64 address pointer
// authentication key recognised by PR_PAC_SET_ENABLED_KEYS. The generic
// authentication key (PR_PAC_APGAKEY) is NOT accepted by this prctl on
// Linux (the kernel returns EINVAL); generic auth applies only to PACGA
// which the VDSO does not use, so it is irrelevant for this fix.
const addressPACKeys = unix.PR_PAC_APIAKEY | unix.PR_PAC_APIBKEY |
	unix.PR_PAC_APDAKEY | unix.PR_PAC_APDBKEY

// disableHostPAC disables host pointer authentication for the calling
// thread. This is necessary on ARM64 because the sentry's Go runtime runs
// inside the KVM guest at Guest EL1, where PAC instructions
// (paciasp/autiasp) behave differently than at Host EL0. When the
// sentry's Go runtime calls a VDSO function such as __kernel_getrandom,
// the VDSO's paciasp signs the return address with whatever PAC state
// the guest happens to have. The subsequent SVC traps to El1_sync, the
// vCPU exits, and bluepillArchExit copies the VDSO PC into the host's
// signal context. After sigreturn resumes the host thread at the VDSO,
// the VDSO epilogue's autiasp tries to verify the return address with
// the host's PAC keys, fails, and raises SIGILL. The gVisor SIGILL
// handler then misinterprets R8 (which contains VDSO state, not a
// vCPU pointer) and crashes with SIGSEGV.
//
// The PAC keys are per-process secrets and cannot be read from
// userspace, so there is no way to synchronise them between Guest EL1
// and Host EL0. Instead, disable the host's PAC keys entirely; that
// turns paciasp/autiasp into NOPs at Host EL0, which removes the
// mismatch without requiring any guest-side coordination.
//
// This is called from loadSegments, which runs every time a vCPU is
// bound to a (potentially new) host thread. Each thread that ever
// hosts a vCPU therefore has PAC disabled before it can trigger the
// bluepill SIGILL path. The prctl is per-thread; calling it again on
// a thread that already has PAC disabled is harmless and idempotent
// at the kernel level.
//
// The prctl return value is intentionally ignored: it returns EINVAL
// on kernels older than 5.13 (which lack PR_PAC_SET_ENABLED_KEYS) and
// on hardware without address authentication, both of which imply the
// crash cannot occur and there is nothing to fix.
//
//go:nosplit
func disableHostPAC() {
	if !checkPAC() {
		return
	}
	hostsyscall.RawSyscallErrno6(
		unix.SYS_PRCTL,
		unix.PR_PAC_SET_ENABLED_KEYS,
		addressPACKeys, // keys to modify
		0,              // 0 = disable, 1 = enable
		0, 0, 0)
}

//go:nosplit
func (c *vCPU) setOneRegister(reg *kvmOneReg) error {
	if errno := hostsyscall.RawSyscallErrno(
		unix.SYS_IOCTL,
		uintptr(c.fd),
		_KVM_SET_ONE_REG,
		uintptr(unsafe.Pointer(reg))); errno != 0 {
		return fmt.Errorf("error setting one register: %v", errno)
	}
	return nil
}

//go:nosplit
func (c *vCPU) getOneRegister(reg *kvmOneReg) unix.Errno {
	return hostsyscall.RawSyscallErrno(
		unix.SYS_IOCTL,
		uintptr(c.fd),
		_KVM_GET_ONE_REG,
		uintptr(unsafe.Pointer(reg)))
}

// SwitchToUser unpacks architectural-details.
func (c *vCPU) SwitchToUser(switchOpts ring0.SwitchOpts, info *linux.SignalInfo) (hostarch.AccessType, error) {
	// Check for canonical addresses.
	if regs := switchOpts.Registers; !ring0.IsCanonical(regs.Pc) {
		return nonCanonical(regs.Pc, int32(unix.SIGSEGV), info)
	} else if !ring0.IsCanonical(regs.Sp) {
		return nonCanonical(regs.Sp, int32(unix.SIGSEGV), info)
	}

	// Assign PCIDs.
	if c.PCIDs != nil {
		var requireFlushPCID bool // Force a flush?
		switchOpts.UserASID, requireFlushPCID = c.PCIDs.Assign(switchOpts.PageTables)
		switchOpts.Flush = switchOpts.Flush || requireFlushPCID
	}

	var vector ring0.Vector
	ttbr0App := switchOpts.PageTables.TTBR0_EL1(false, 0)
	c.SetTtbr0App(uintptr(ttbr0App))

	// Full context-switch supporting for Arm64.
	// The Arm64 user-mode execution state consists of:
	// x0-x30
	// PC, SP, PSTATE
	// V0-V31: 32 128-bit registers for floating point, and simd
	// FPSR, FPCR
	// TPIDR_EL0, used for TLS
	appRegs := switchOpts.Registers
	c.SetAppAddr(ring0.KernelStartAddress | uintptr(unsafe.Pointer(appRegs)))

	c.switchingToUser.Store(true)

	entersyscall()
	bluepill(c)
	vector = c.CPU.SwitchToUser(switchOpts)
	exitsyscall()

	c.switchingToUser.Store(false)

	switch vector {
	case ring0.Syscall:
		// Fast path: system call executed.
		return hostarch.NoAccess, nil
	case ring0.PageFault:
		return c.fault(int32(unix.SIGSEGV), info)
	case ring0.El0ErrNMI:
		return c.fault(int32(unix.SIGBUS), info)
	case ring0.Vector(bounce): // ring0.VirtualizationException.
		return hostarch.NoAccess, platform.ErrContextInterrupt
	case ring0.El0SyncUndef,
		ring0.El0SyncInv:
		return c.fault(int32(unix.SIGILL), info)
	case ring0.El0SyncDbg:
		*info = linux.SignalInfo{
			Signo: int32(unix.SIGTRAP),
			Code:  1, // TRAP_BRKPT (breakpoint).
		}
		info.SetAddr(switchOpts.Registers.Pc) // Include address.
		return hostarch.AccessType{}, platform.ErrContextSignal
	case ring0.El0SyncSpPc:
		*info = linux.SignalInfo{
			Signo: int32(unix.SIGBUS),
			Code:  2, // BUS_ADRERR (physical address does not exist).
		}
		return hostarch.NoAccess, platform.ErrContextSignal
	case ring0.El0SyncSys,
		ring0.El0SyncWfx:
		return hostarch.NoAccess, nil // skip for now.
	default:
		panic(fmt.Sprintf("unexpected vector: 0x%x", vector))
	}

}

//go:nosplit
func seccompMmapSyscall(context unsafe.Pointer) (uintptr, uintptr, unix.Errno) {
	ctx := bluepillArchContext(context)

	// MAP_DENYWRITE is deprecated and ignored by kernel. We use it only for seccomp filters.
	addr, e := hostsyscall.RawSyscall6(uintptr(ctx.Regs[8]), uintptr(ctx.Regs[0]), uintptr(ctx.Regs[1]),
		uintptr(ctx.Regs[2]), uintptr(ctx.Regs[3])|unix.MAP_DENYWRITE, uintptr(ctx.Regs[4]), uintptr(ctx.Regs[5]))
	ctx.Regs[0] = uint64(addr)

	return addr, uintptr(ctx.Regs[1]), unix.Errno(e)
}