mirror of
https://github.com/hanzoai/visor.git
synced 2026-09-24 17:04:14 +00:00
Konstantin BogomolovandgVisor bot
d5bf6cb7e4
Initialize FPU and XCR0 in assembly before entering Go code, switch %rip to the lower half before calling kernel syscall and exception trampolines. In recent Go toolchains, the compiler-generated ABI0 wrappers that bridge assembly and internal ABI Go code zero X15. The wrapper performs a RIP-relative load of runtime.x86HasAVX to see if AVX is available on the host, and executes VXORPS. This introduces two requirements in ring0: 1. When booting a vCPU, XCR0 only enables x87 state by default. Executing VXORPS in startGo.abi0 triggers a #UD fault if AVX is not enabled in XCR0. Moving fninit and xsetbv into assembly before calling startGo ensures AVX state is enabled before the wrapper runs. 2. The syscall and exception entrypoints execute in the upper half of the virtual address space where data pages are not mapped. The RIP-relative load of runtime.x86HasAVX faults unless we transition %rip to the lower half via jumpToUser before calling into Go trampolines. PiperOrigin-RevId: 974188468
780 lines
27 KiB
ArmAsm
780 lines
27 KiB
ArmAsm
// Copyright 2018 The gVisor Authors.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
#include "funcdata.h"
|
|
#include "textflag.h"
|
|
|
|
// CPU offsets.
|
|
#define CPU_REGISTERS 72 // +checkoffset . CPU.registers
|
|
#define CPU_FPU_STATE 288 // +checkoffset . CPU.floatingPointState
|
|
#define CPU_ARCH_STATE 16 // +checkoffset . CPU.CPUArchState
|
|
#define CPU_ERROR_CODE CPU_ARCH_STATE+0 // +checkoffset . CPUArchState.errorCode
|
|
#define CPU_ERROR_TYPE CPU_ARCH_STATE+8 // +checkoffset . CPUArchState.errorType
|
|
#define CPU_VECTOR CPU_ARCH_STATE+16 // +checkoffset . CPUArchState.vector
|
|
#define CPU_FAULT_ADDR CPU_ARCH_STATE+24 // +checkoffset . CPUArchState.faultAddr
|
|
#define CPU_ENTRY CPU_ARCH_STATE+32 // +checkoffset . CPUArchState.kernelEntry
|
|
#define CPU_APP_GS_BASE CPU_ARCH_STATE+40 // +checkoffset . CPUArchState.appGsBase
|
|
#define CPU_HAS_XSAVE CPU_ARCH_STATE+48 // +checkoffset . CPUArchState.hasXSAVE
|
|
#define CPU_HAS_XSAVEOPT CPU_ARCH_STATE+49 // +checkoffset . CPUArchState.hasXSAVEOPT
|
|
#define CPU_HAS_FSGSBASE CPU_ARCH_STATE+50 // +checkoffset . CPUArchState.hasFSGSBASE
|
|
#define CPU_XCR0 CPU_ARCH_STATE+52 // +checkoffset . CPUArchState.xcr0Eax
|
|
|
|
#define ENTRY_SCRATCH0 256 // +checkoffset . kernelEntry.scratch0
|
|
#define ENTRY_STACK_TOP 264 // +checkoffset . kernelEntry.stackTop
|
|
#define ENTRY_CPU_SELF 272 // +checkoffset . kernelEntry.cpuSelf
|
|
#define ENTRY_KERNEL_CR3 280 // +checkoffset . kernelEntry.kernelCR3
|
|
#define ENTRY_ENABLE_VMCALL 288 // +checkoffset . kernelEntry.enableVMCALL
|
|
|
|
// Bits.
|
|
#define _RFLAGS_IF 512 // +checkconst . _RFLAGS_IF
|
|
#define _RFLAGS_IOPL0 4096 // +checkconst . _RFLAGS_IOPL0
|
|
#define _KERNEL_FLAGS 2 // +checkconst . KernelFlagsSet
|
|
|
|
// Vectors.
|
|
#define DivideByZero 0 // +checkconst . DivideByZero
|
|
#define Debug 1 // +checkconst . Debug
|
|
#define NMI 2 // +checkconst . NMI
|
|
#define Breakpoint 3 // +checkconst . Breakpoint
|
|
#define Overflow 4 // +checkconst . Overflow
|
|
#define BoundRangeExceeded 5 // +checkconst . BoundRangeExceeded
|
|
#define InvalidOpcode 6 // +checkconst . InvalidOpcode
|
|
#define DeviceNotAvailable 7 // +checkconst . DeviceNotAvailable
|
|
#define DoubleFault 8 // +checkconst . DoubleFault
|
|
#define CoprocessorSegmentOverrun 9 // +checkconst . CoprocessorSegmentOverrun
|
|
#define InvalidTSS 10 // +checkconst . InvalidTSS
|
|
#define SegmentNotPresent 11 // +checkconst . SegmentNotPresent
|
|
#define StackSegmentFault 12 // +checkconst . StackSegmentFault
|
|
#define GeneralProtectionFault 13 // +checkconst . GeneralProtectionFault
|
|
#define PageFault 14 // +checkconst . PageFault
|
|
#define X87FloatingPointException 16 // +checkconst . X87FloatingPointException
|
|
#define AlignmentCheck 17 // +checkconst . AlignmentCheck
|
|
#define MachineCheck 18 // +checkconst . MachineCheck
|
|
#define SIMDFloatingPointException 19 // +checkconst . SIMDFloatingPointException
|
|
#define VirtualizationException 20 // +checkconst . VirtualizationException
|
|
#define SecurityException 30 // +checkconst . SecurityException
|
|
#define SyscallInt80 128 // +checkconst . SyscallInt80
|
|
#define Syscall 256 // +checkconst . Syscall
|
|
|
|
#define SyscallExit 60 // +checkconst . SyscallExit
|
|
#define SyscallExitGroup 231 // +checkconst . SyscallExitGroup
|
|
#define SyscallRedPill 4294967295 // +checkconst . SyscallRedPill
|
|
|
|
#define CPUIntel 0 // +checkconst . CPUIntel
|
|
|
|
#define PTRACE_R15 0 // +checkoffset linux PtraceRegs.R15
|
|
#define PTRACE_R14 8 // +checkoffset linux PtraceRegs.R14
|
|
#define PTRACE_R13 16 // +checkoffset linux PtraceRegs.R13
|
|
#define PTRACE_R12 24 // +checkoffset linux PtraceRegs.R12
|
|
#define PTRACE_RBP 32 // +checkoffset linux PtraceRegs.Rbp
|
|
#define PTRACE_RBX 40 // +checkoffset linux PtraceRegs.Rbx
|
|
#define PTRACE_R11 48 // +checkoffset linux PtraceRegs.R11
|
|
#define PTRACE_R10 56 // +checkoffset linux PtraceRegs.R10
|
|
#define PTRACE_R9 64 // +checkoffset linux PtraceRegs.R9
|
|
#define PTRACE_R8 72 // +checkoffset linux PtraceRegs.R8
|
|
#define PTRACE_RAX 80 // +checkoffset linux PtraceRegs.Rax
|
|
#define PTRACE_RCX 88 // +checkoffset linux PtraceRegs.Rcx
|
|
#define PTRACE_RDX 96 // +checkoffset linux PtraceRegs.Rdx
|
|
#define PTRACE_RSI 104 // +checkoffset linux PtraceRegs.Rsi
|
|
#define PTRACE_RDI 112 // +checkoffset linux PtraceRegs.Rdi
|
|
#define PTRACE_ORIGRAX 120 // +checkoffset linux PtraceRegs.Orig_rax
|
|
#define PTRACE_RIP 128 // +checkoffset linux PtraceRegs.Rip
|
|
#define PTRACE_CS 136 // +checkoffset linux PtraceRegs.Cs
|
|
#define PTRACE_FLAGS 144 // +checkoffset linux PtraceRegs.Eflags
|
|
#define PTRACE_RSP 152 // +checkoffset linux PtraceRegs.Rsp
|
|
#define PTRACE_SS 160 // +checkoffset linux PtraceRegs.Ss
|
|
#define PTRACE_FS_BASE 168 // +checkoffset linux PtraceRegs.Fs_base
|
|
#define PTRACE_GS_BASE 176 // +checkoffset linux PtraceRegs.Gs_base
|
|
|
|
#define XCR0_EAX 231 // +checkconst fpu SupportedXFeatureStates
|
|
#define XCR0_EDX 0
|
|
|
|
// Saves a register set.
|
|
//
|
|
// This is a macro because it may need to executed in contents where a stack is
|
|
// not available for calls.
|
|
//
|
|
// The following registers are not saved: AX, SP, IP, FLAGS, all segments.
|
|
#define REGISTERS_SAVE(reg, offset) \
|
|
MOVQ R15, offset+PTRACE_R15(reg); \
|
|
MOVQ R14, offset+PTRACE_R14(reg); \
|
|
MOVQ R13, offset+PTRACE_R13(reg); \
|
|
MOVQ R12, offset+PTRACE_R12(reg); \
|
|
MOVQ BP, offset+PTRACE_RBP(reg); \
|
|
MOVQ BX, offset+PTRACE_RBX(reg); \
|
|
MOVQ CX, offset+PTRACE_RCX(reg); \
|
|
MOVQ DX, offset+PTRACE_RDX(reg); \
|
|
MOVQ R11, offset+PTRACE_R11(reg); \
|
|
MOVQ R10, offset+PTRACE_R10(reg); \
|
|
MOVQ R9, offset+PTRACE_R9(reg); \
|
|
MOVQ R8, offset+PTRACE_R8(reg); \
|
|
MOVQ SI, offset+PTRACE_RSI(reg); \
|
|
MOVQ DI, offset+PTRACE_RDI(reg);
|
|
|
|
// Loads a register set.
|
|
//
|
|
// This is a macro because it may need to executed in contents where a stack is
|
|
// not available for calls.
|
|
//
|
|
// The following registers are not loaded: AX, SP, IP, FLAGS, all segments.
|
|
#define REGISTERS_LOAD(reg, offset) \
|
|
MOVQ offset+PTRACE_R15(reg), R15; \
|
|
MOVQ offset+PTRACE_R14(reg), R14; \
|
|
MOVQ offset+PTRACE_R13(reg), R13; \
|
|
MOVQ offset+PTRACE_R12(reg), R12; \
|
|
MOVQ offset+PTRACE_RBP(reg), BP; \
|
|
MOVQ offset+PTRACE_RBX(reg), BX; \
|
|
MOVQ offset+PTRACE_RCX(reg), CX; \
|
|
MOVQ offset+PTRACE_RDX(reg), DX; \
|
|
MOVQ offset+PTRACE_R11(reg), R11; \
|
|
MOVQ offset+PTRACE_R10(reg), R10; \
|
|
MOVQ offset+PTRACE_R9(reg), R9; \
|
|
MOVQ offset+PTRACE_R8(reg), R8; \
|
|
MOVQ offset+PTRACE_RSI(reg), SI; \
|
|
MOVQ offset+PTRACE_RDI(reg), DI;
|
|
|
|
// WRITE_CR3() writes the given CR3 value.
|
|
//
|
|
// The code corresponds to:
|
|
//
|
|
// mov %rax, %cr3
|
|
//
|
|
#define WRITE_CR3() \
|
|
BYTE $0x0f; BYTE $0x22; BYTE $0xd8;
|
|
|
|
// SWAP_GS swaps the kernel GS (CPU).
|
|
#define SWAP_GS() \
|
|
BYTE $0x0F; BYTE $0x01; BYTE $0xf8;
|
|
|
|
// IRET returns from an interrupt frame.
|
|
#define IRET() \
|
|
BYTE $0x48; BYTE $0xcf;
|
|
|
|
// SYSRET64 executes the sysret instruction.
|
|
#define SYSRET64() \
|
|
BYTE $0x48; BYTE $0x0f; BYTE $0x07;
|
|
|
|
// LOAD_KERNEL_STACK loads the kernel stack.
|
|
#define LOAD_KERNEL_STACK(entry) \
|
|
MOVQ ENTRY_STACK_TOP(entry), SP;
|
|
|
|
// VMCALL do vmcall/vmmcal instruction
|
|
#define VMCALL() \
|
|
CMPQ ·CPUVendor(SB), $CPUIntel; \
|
|
JE 2(PC); \
|
|
JMP 5(PC); \ // vmmcall and vmcall will be treated as 3 independent instructions
|
|
BYTE $0x0F; BYTE $0x01; BYTE $0xC1; \
|
|
JMP 4(PC); \ // vmmcall and vmcall will be treated as 3 independent instructions
|
|
BYTE $0x0F; BYTE $0x01; BYTE $0xD9;
|
|
|
|
// ADDR_OF_FUNC defines a function named 'name' that returns the address of
|
|
// 'symbol'.
|
|
#define ADDR_OF_FUNC(name, symbol) \
|
|
TEXT name,$0-8; \
|
|
MOVQ $symbol, AX; \
|
|
MOVQ AX, ret+0(FP); \
|
|
RET
|
|
|
|
// See kernel.go.
|
|
TEXT ·Halt(SB),NOSPLIT|NOFRAME,$0
|
|
HLT
|
|
RET
|
|
|
|
// See kernel_amd64.go.
|
|
TEXT ·HaltAndWriteFSBase(SB),NOSPLIT,$8-8
|
|
HLT
|
|
|
|
// Restore FS_BASE.
|
|
MOVQ regs+0(FP), AX
|
|
MOVQ PTRACE_FS_BASE(AX), AX
|
|
|
|
PUSHQ AX // First argument (FS_BASE)
|
|
CALL ·writeFS(SB)
|
|
POPQ AX
|
|
|
|
RET
|
|
|
|
// jumpToKernel changes execution to the kernel address space.
|
|
//
|
|
// This works by changing the return value to the kernel version.
|
|
TEXT ·jumpToKernel(SB),NOSPLIT|NOFRAME,$0
|
|
MOVQ 0(SP), AX
|
|
ORQ ·KernelStartAddress(SB), AX // Future return value.
|
|
MOVQ AX, 0(SP)
|
|
RET
|
|
|
|
// jumpToUser changes execution to the user address space.
|
|
//
|
|
// This works by changing the return value to the user version.
|
|
TEXT ·jumpToUser(SB),NOSPLIT|NOFRAME,$0
|
|
// N.B. we can't access KernelStartAddress from the upper half (data
|
|
// pages not available), so just naively clear all the upper bits.
|
|
// We are assuming a 47-bit virtual address space.
|
|
MOVQ $0x00007fffffffffff, AX
|
|
MOVQ 0(SP), BX
|
|
ANDQ BX, AX // Future return value.
|
|
MOVQ AX, 0(SP)
|
|
RET
|
|
|
|
// See kernel_amd64.go.
|
|
//
|
|
// The 16-byte frame size is for the saved values of MXCSR and the x87 control
|
|
// word.
|
|
TEXT ·doSwitchToUser(SB),NOSPLIT,$16-48
|
|
// We are passed pointers to heap objects, but do not store them in our
|
|
// local frame.
|
|
NO_LOCAL_POINTERS
|
|
|
|
// MXCSR and the x87 control word are the only floating point state
|
|
// that is callee-save and thus we must save.
|
|
STMXCSR mxcsr-0(SP)
|
|
FSTCW cw-8(SP)
|
|
|
|
// Restore application floating point state.
|
|
MOVQ cpu+0(FP), SI
|
|
MOVQ fpState+16(FP), DI
|
|
MOVB ·hasXSAVE(SB), BX
|
|
TESTB BX, BX
|
|
JZ no_xrstor
|
|
// Use xrstor to restore all available fp state.
|
|
MOVL $XCR0_EAX, AX
|
|
MOVL $XCR0_EDX, DX
|
|
BYTE $0x48; BYTE $0x0f; BYTE $0xae; BYTE $0x2f // XRSTOR64 0(DI)
|
|
JMP fprestore_done
|
|
no_xrstor:
|
|
// Fall back to fxrstor if xsave is not available.
|
|
FXRSTOR64 0(DI)
|
|
fprestore_done:
|
|
|
|
// Set application GS.
|
|
MOVQ regs+8(FP), R8
|
|
SWAP_GS()
|
|
MOVQ PTRACE_GS_BASE(R8), AX
|
|
CMPQ AX, CPU_APP_GS_BASE(SI)
|
|
JE skip_gs
|
|
MOVQ AX, CPU_APP_GS_BASE(SI)
|
|
PUSHQ AX
|
|
CALL ·writeGS(SB)
|
|
POPQ AX
|
|
skip_gs:
|
|
// Call sysret() or iret().
|
|
MOVQ userCR3+24(FP), CX
|
|
MOVQ needIRET+32(FP), R9
|
|
ADDQ $-32, SP
|
|
MOVQ SI, 0(SP) // cpu
|
|
MOVQ R8, 8(SP) // regs
|
|
MOVQ CX, 16(SP) // userCR3
|
|
TESTQ R9, R9
|
|
JNZ do_iret
|
|
CALL ·sysret(SB)
|
|
JMP done_sysret_or_iret
|
|
do_iret:
|
|
CALL ·iret(SB)
|
|
done_sysret_or_iret:
|
|
MOVQ 24(SP), AX // vector
|
|
ADDQ $32, SP
|
|
MOVQ AX, ret+40(FP)
|
|
|
|
// Save application floating point state.
|
|
MOVQ fpState+16(FP), DI
|
|
MOVB ·hasXSAVE(SB), BX
|
|
MOVB ·hasXSAVEOPT(SB), CX
|
|
TESTB BX, BX
|
|
JZ no_xsave
|
|
// Use xsave/xsaveopt to save all extended state.
|
|
MOVL $XCR0_EAX, AX
|
|
MOVL $XCR0_EDX, DX
|
|
TESTB CX, CX
|
|
JZ no_xsaveopt
|
|
BYTE $0x48; BYTE $0x0f; BYTE $0xae; BYTE $0x37; // XSAVEOPT64 0(DI)
|
|
JMP fpsave_done
|
|
no_xsaveopt:
|
|
BYTE $0x48; BYTE $0x0f; BYTE $0xae; BYTE $0x27; // XSAVE64 0(DI)
|
|
JMP fpsave_done
|
|
no_xsave:
|
|
FXSAVE64 0(DI)
|
|
fpsave_done:
|
|
|
|
// Restore MXCSR and the x87 control word after one of the two floating
|
|
// point save cases above, to ensure the application versions are saved
|
|
// before being clobbered here.
|
|
LDMXCSR mxcsr-0(SP)
|
|
|
|
// FLDCW is a "waiting" x87 instruction, meaning it checks for pending
|
|
// unmasked exceptions before executing. Thus if userspace has unmasked
|
|
// an exception and has one pending, it can be raised by FLDCW even
|
|
// though the new control word will mask exceptions. To prevent this,
|
|
// we must first clear pending exceptions (which will be restored by
|
|
// XRSTOR, et al).
|
|
BYTE $0xDB; BYTE $0xE2; // FNCLEX
|
|
FLDCW cw-8(SP)
|
|
|
|
RET
|
|
|
|
// See entry_amd64.go.
|
|
TEXT ·sysret(SB),NOSPLIT|NOFRAME,$0-32
|
|
// Set application FS. We can't do this in Go because Go code needs FS.
|
|
MOVQ regs+8(FP), AX
|
|
MOVQ PTRACE_FS_BASE(AX), AX
|
|
|
|
PUSHQ AX
|
|
CALL ·writeFS(SB)
|
|
POPQ AX
|
|
|
|
CALL ·jumpToKernel(SB)
|
|
// Save original state and stack. sysenter() or exception()
|
|
// from APP(gr3) will switch to this stack, set the return
|
|
// value (vector: 32(SP)) and then do RET, which will also
|
|
// automatically return to the lower half.
|
|
MOVQ cpu+0(FP), BX
|
|
MOVQ regs+8(FP), AX
|
|
MOVQ userCR3+16(FP), CX
|
|
MOVQ SP, CPU_REGISTERS+PTRACE_RSP(BX)
|
|
MOVQ BP, CPU_REGISTERS+PTRACE_RBP(BX)
|
|
MOVQ AX, CPU_REGISTERS+PTRACE_RAX(BX)
|
|
|
|
// save SP AX userCR3 on the kernel stack.
|
|
MOVQ CPU_ENTRY(BX), BX
|
|
LOAD_KERNEL_STACK(BX)
|
|
PUSHQ PTRACE_RSP(AX)
|
|
PUSHQ PTRACE_RAX(AX)
|
|
PUSHQ CX
|
|
|
|
// Restore user register state.
|
|
REGISTERS_LOAD(AX, 0)
|
|
MOVQ PTRACE_RIP(AX), CX // Needed for SYSRET.
|
|
MOVQ PTRACE_FLAGS(AX), R11 // Needed for SYSRET.
|
|
|
|
// restore userCR3, AX, SP.
|
|
POPQ AX // Get userCR3.
|
|
WRITE_CR3() // Switch to userCR3.
|
|
POPQ AX // Restore AX.
|
|
POPQ SP // Restore SP.
|
|
SYSRET64()
|
|
// sysenter or exception will write our return value and return to our
|
|
// caller.
|
|
|
|
// See entry_amd64.go.
|
|
TEXT ·iret(SB),NOSPLIT|NOFRAME,$0-32
|
|
// Set application FS. We can't do this in Go because Go code needs FS.
|
|
MOVQ regs+8(FP), AX
|
|
MOVQ PTRACE_FS_BASE(AX), AX
|
|
|
|
PUSHQ AX // First argument (FS_BASE)
|
|
CALL ·writeFS(SB)
|
|
POPQ AX
|
|
|
|
CALL ·jumpToKernel(SB)
|
|
// Save original state and stack. sysenter() or exception()
|
|
// from APP(gr3) will switch to this stack, set the return
|
|
// value (vector: 32(SP)) and then do RET, which will also
|
|
// automatically return to the lower half.
|
|
MOVQ cpu+0(FP), BX
|
|
MOVQ regs+8(FP), AX
|
|
MOVQ userCR3+16(FP), CX
|
|
MOVQ SP, CPU_REGISTERS+PTRACE_RSP(BX)
|
|
MOVQ BP, CPU_REGISTERS+PTRACE_RBP(BX)
|
|
MOVQ AX, CPU_REGISTERS+PTRACE_RAX(BX)
|
|
|
|
// Build an IRET frame & restore state.
|
|
MOVQ CPU_ENTRY(BX), BX
|
|
LOAD_KERNEL_STACK(BX)
|
|
PUSHQ PTRACE_SS(AX)
|
|
PUSHQ PTRACE_RSP(AX)
|
|
PUSHQ PTRACE_FLAGS(AX)
|
|
PUSHQ PTRACE_CS(AX)
|
|
PUSHQ PTRACE_RIP(AX)
|
|
PUSHQ PTRACE_RAX(AX) // Save AX on kernel stack.
|
|
PUSHQ CX // Save userCR3 on kernel stack.
|
|
REGISTERS_LOAD(AX, 0) // Restore most registers.
|
|
POPQ AX // Get userCR3.
|
|
WRITE_CR3() // Switch to userCR3.
|
|
POPQ AX // Restore AX.
|
|
IRET()
|
|
// sysenter or exception will write our return value and return to our
|
|
// caller.
|
|
|
|
// See entry_amd64.go.
|
|
TEXT ·resume(SB),NOSPLIT|NOFRAME,$0
|
|
// See iret, above.
|
|
MOVQ ENTRY_CPU_SELF(GS), AX // Load vCPU.
|
|
PUSHQ CPU_REGISTERS+PTRACE_SS(AX)
|
|
PUSHQ CPU_REGISTERS+PTRACE_RSP(AX)
|
|
PUSHQ CPU_REGISTERS+PTRACE_FLAGS(AX)
|
|
PUSHQ CPU_REGISTERS+PTRACE_CS(AX)
|
|
PUSHQ CPU_REGISTERS+PTRACE_RIP(AX)
|
|
REGISTERS_LOAD(AX, CPU_REGISTERS)
|
|
MOVQ CPU_REGISTERS+PTRACE_RAX(AX), AX
|
|
IRET()
|
|
|
|
// See entry_amd64.go.
|
|
TEXT ·start(SB),NOSPLIT|NOFRAME,$0
|
|
// N.B. This is the vCPU entrypoint. It is not called from Go code and
|
|
// thus pushes and pops values on the stack until calling into Go
|
|
// (startGo) because we aren't usually a typical Go assembly frame.
|
|
PUSHQ $0x0 // Previous frame pointer.
|
|
MOVQ SP, BP // Set frame pointer.
|
|
PUSHQ AX // Save CPU.
|
|
|
|
// Set up environment required by Go before calling startGo: Go needs
|
|
// FS_BASE and floating point initialized.
|
|
MOVQ CPU_REGISTERS+PTRACE_FS_BASE(AX), BX
|
|
PUSHQ BX // First argument (FS_BASE)
|
|
CALL ·writeFS(SB)
|
|
POPQ BX
|
|
|
|
MOVQ CPU_APP_GS_BASE(AX),BX
|
|
PUSHQ BX
|
|
CALL ·writeGS(SB)
|
|
POPQ BX
|
|
SWAP_GS()
|
|
|
|
// Initialize FPU state before calling any Go code.
|
|
BYTE $0xdb; BYTE $0xe3 // fninit
|
|
|
|
// Sync XCR0 if XSAVE is enabled.
|
|
MOVQ 0(SP), AX // Pointer to CPU.
|
|
CMPB CPU_HAS_XSAVE(AX), $0
|
|
JE no_xsetbv_start
|
|
MOVL CPU_XCR0(AX), AX
|
|
MOVL $0, DX
|
|
MOVL $0, CX
|
|
BYTE $0x0f; BYTE $0x01; BYTE $0xd1 // xsetbv
|
|
no_xsetbv_start:
|
|
|
|
// First argument (CPU) already at bottom of stack.
|
|
CALL ·startGo(SB) // Call Go hook.
|
|
JMP ·resume(SB) // Restore to registers.
|
|
|
|
ADDR_OF_FUNC(·AddrOfStart(SB), ·start(SB));
|
|
|
|
// See entry_amd64.go.
|
|
TEXT ·sysenter(SB),NOSPLIT|NOFRAME,$0
|
|
// _RFLAGS_IOPL0 is always set in the user mode and it is never set in
|
|
// the kernel mode. See the comment of UserFlagsSet for more details.
|
|
TESTL $_RFLAGS_IOPL0, R11
|
|
JZ kernel
|
|
user:
|
|
SWAP_GS()
|
|
MOVQ AX, ENTRY_SCRATCH0(GS) // Save user AX on scratch.
|
|
MOVQ ENTRY_KERNEL_CR3(GS), AX // Get kernel cr3 on AX.
|
|
WRITE_CR3() // Switch to kernel cr3.
|
|
|
|
MOVQ ENTRY_CPU_SELF(GS), AX // Load vCPU.
|
|
MOVQ CPU_REGISTERS+PTRACE_RAX(AX), AX // Get user regs.
|
|
REGISTERS_SAVE(AX, 0) // Save all except IP, FLAGS, SP, AX.
|
|
MOVQ CX, PTRACE_RIP(AX)
|
|
MOVQ R11, PTRACE_FLAGS(AX)
|
|
MOVQ SP, PTRACE_RSP(AX)
|
|
MOVQ ENTRY_SCRATCH0(GS), CX // Load saved user AX value.
|
|
MOVQ CX, PTRACE_RAX(AX) // Save everything else.
|
|
MOVQ CX, PTRACE_ORIGRAX(AX)
|
|
|
|
CMPB CPU_HAS_FSGSBASE(GS), $1
|
|
JNE sysenter_skip_gs
|
|
SWAP_GS()
|
|
BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xae; BYTE $0xcb; // rdgsbase rbx
|
|
MOVQ BX, PTRACE_GS_BASE(AX)
|
|
SWAP_GS()
|
|
|
|
sysenter_skip_gs:
|
|
MOVQ ENTRY_CPU_SELF(GS), AX // Load vCPU.
|
|
MOVQ CPU_REGISTERS+PTRACE_RSP(AX), SP // Get stacks.
|
|
MOVQ $0, CPU_ERROR_CODE(AX) // Clear error code.
|
|
MOVQ $1, CPU_ERROR_TYPE(AX) // Set error type to user.
|
|
|
|
CALL ·jumpToUser(SB)
|
|
|
|
// Restore kernel FS_BASE.
|
|
MOVQ ENTRY_CPU_SELF(GS), AX // Load vCPU.
|
|
MOVQ CPU_REGISTERS+PTRACE_FS_BASE(AX), BX
|
|
|
|
PUSHQ BX // First argument (FS_BASE)
|
|
CALL ·writeFS(SB)
|
|
POPQ BX
|
|
|
|
MOVQ ENTRY_CPU_SELF(GS), AX // Load vCPU.
|
|
|
|
// Return to the kernel, where the frame is:
|
|
//
|
|
// vector (sp+32)
|
|
// userCR3 (sp+24)
|
|
// regs (sp+16)
|
|
// cpu (sp+8)
|
|
// vcpu.Switch (sp+0)
|
|
//
|
|
MOVQ CPU_REGISTERS+PTRACE_RBP(AX), BP // Original base pointer.
|
|
MOVQ $Syscall, 32(SP) // Output vector.
|
|
RET
|
|
|
|
kernel:
|
|
// Handle any syscalls from GR0 in HR3 when EnableVMCALL is false.
|
|
// Currently there are 2 use cases:
|
|
// 1. Using KVM platform.
|
|
// 2. Upgrading SlimVM platform. This is one such method to return M to
|
|
// user mode (HR3) for upgrading platform.
|
|
CMPQ ENTRY_ENABLE_VMCALL(GS), $0
|
|
JE hr3_do_syscall
|
|
|
|
CMPQ AX, $SyscallRedPill
|
|
JE hr3_do_syscall
|
|
|
|
CMPQ AX, $SyscallExit
|
|
JE hr3_do_syscall
|
|
|
|
CMPQ AX, $SyscallExitGroup
|
|
JE hr3_do_syscall
|
|
|
|
vmcall:
|
|
// handle syscall from GR0 in host kernel
|
|
// copy from "handle system calls from G0" part of __dune_syscall in libdune/dune.S
|
|
PUSHQ R11
|
|
POPFQ
|
|
|
|
CMPQ AX, $158 // arch_prctl syscall
|
|
JNE 3(PC)
|
|
CMPQ DI, $0x1002 //ARCH_SET_FS
|
|
JE arch_prctl_vmcall
|
|
|
|
VMCALL()
|
|
JMP *CX
|
|
|
|
arch_prctl_vmcall:
|
|
VMCALL()
|
|
CMPQ AX, $0
|
|
JNE 2(PC)
|
|
MOVQ SI, CPU_REGISTERS+PTRACE_FS_BASE(GS)
|
|
JMP *CX
|
|
|
|
hr3_do_syscall:
|
|
// We can't restore the original stack, but we can access the registers
|
|
// in the CPU state directly. No need for temporary juggling.
|
|
MOVQ AX, ENTRY_SCRATCH0(GS)
|
|
MOVQ ENTRY_CPU_SELF(GS), AX // Load vCPU.
|
|
REGISTERS_SAVE(AX, CPU_REGISTERS)
|
|
MOVQ CX, CPU_REGISTERS+PTRACE_RIP(AX)
|
|
MOVQ R11, CPU_REGISTERS+PTRACE_FLAGS(AX)
|
|
MOVQ SP, CPU_REGISTERS+PTRACE_RSP(AX)
|
|
MOVQ ENTRY_SCRATCH0(GS), BX
|
|
MOVQ BX, CPU_REGISTERS+PTRACE_ORIGRAX(AX)
|
|
MOVQ BX, CPU_REGISTERS+PTRACE_RAX(AX)
|
|
MOVQ $0, CPU_ERROR_CODE(AX) // Clear error code.
|
|
MOVQ $0, CPU_ERROR_TYPE(AX) // Set error type to kernel.
|
|
MOVQ $0xffffffffffffffff, CPU_VECTOR(AX) // Set error type to kernel.
|
|
|
|
// Save floating point state. CPU.floatingPointState is a slice, so the
|
|
// first word of CPU.floatingPointState is a pointer to the destination
|
|
// array.
|
|
MOVQ CPU_FPU_STATE(AX), DI
|
|
MOVB CPU_HAS_XSAVE(AX), BX
|
|
MOVB CPU_HAS_XSAVEOPT(AX), CX
|
|
TESTB BX, BX
|
|
JZ no_xsave
|
|
// Use xsave/xsaveopt to save all extended state.
|
|
MOVL $XCR0_EAX, AX
|
|
MOVL $XCR0_EDX, DX
|
|
TESTB CX, CX
|
|
JZ no_xsaveopt
|
|
BYTE $0x48; BYTE $0x0f; BYTE $0xae; BYTE $0x37; // XSAVEOPT64 0(DI)
|
|
JMP fpsave_done
|
|
no_xsaveopt:
|
|
BYTE $0x48; BYTE $0x0f; BYTE $0xae; BYTE $0x27; // XSAVE64 0(DI)
|
|
JMP fpsave_done
|
|
no_xsave:
|
|
FXSAVE64 0(DI)
|
|
fpsave_done:
|
|
|
|
// Call the syscall trampoline.
|
|
LOAD_KERNEL_STACK(GS)
|
|
MOVQ ENTRY_CPU_SELF(GS), AX // AX contains the vCPU.
|
|
PUSHQ AX // First argument (vCPU).
|
|
CALL ·jumpToUser(SB)
|
|
CALL ·kernelSyscall(SB) // Call the trampoline.
|
|
POPQ AX // Pop vCPU.
|
|
|
|
// We only trigger a bluepill entry in the bluepill function, and can
|
|
// therefore be guaranteed that there is no floating point state to be
|
|
// loaded on resuming from halt.
|
|
JMP ·resume(SB)
|
|
|
|
ADDR_OF_FUNC(·addrOfSysenter(SB), ·sysenter(SB));
|
|
|
|
// exception is a generic exception handler.
|
|
//
|
|
// There are two cases handled:
|
|
//
|
|
// 1) An exception in kernel mode: this results in saving the state at the time
|
|
// of the exception and calling the defined hook.
|
|
//
|
|
// 2) An exception in guest mode: the original kernel frame is restored, and
|
|
// the vector & error codes are pushed as return values.
|
|
//
|
|
// See below for the stubs that call exception.
|
|
TEXT ·exception(SB),NOSPLIT|NOFRAME,$0
|
|
// Determine whether the exception occurred in kernel mode or user
|
|
// mode, based on the flags. We expect the following stack:
|
|
//
|
|
// SS (sp+48)
|
|
// SP (sp+40)
|
|
// FLAGS (sp+32)
|
|
// CS (sp+24)
|
|
// IP (sp+16)
|
|
// ERROR_CODE (sp+8)
|
|
// VECTOR (sp+0)
|
|
//
|
|
TESTL $_RFLAGS_IOPL0, 32(SP)
|
|
JZ kernel
|
|
|
|
user:
|
|
SWAP_GS()
|
|
ADDQ $-8, SP // Adjust for flags.
|
|
MOVQ $_KERNEL_FLAGS, 0(SP); BYTE $0x9d; // Reset flags (POPFQ).
|
|
PUSHQ AX // Save user AX on stack.
|
|
MOVQ ENTRY_KERNEL_CR3(GS), AX // Get kernel cr3 on AX.
|
|
WRITE_CR3() // Switch to kernel cr3.
|
|
|
|
MOVQ ENTRY_CPU_SELF(GS), AX // Load vCPU.
|
|
MOVQ CPU_REGISTERS+PTRACE_RAX(AX), AX // Get user regs.
|
|
REGISTERS_SAVE(AX, 0) // Save all except IP, FLAGS, SP, AX.
|
|
POPQ BX // Restore original AX.
|
|
MOVQ BX, PTRACE_RAX(AX) // Save it.
|
|
MOVQ BX, PTRACE_ORIGRAX(AX)
|
|
CMPB CPU_HAS_FSGSBASE(GS), $1
|
|
JNE exception_skip_gs
|
|
SWAP_GS()
|
|
BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xae; BYTE $0xcb; // rdgsbase rbx
|
|
MOVQ BX, PTRACE_GS_BASE(AX)
|
|
SWAP_GS()
|
|
exception_skip_gs:
|
|
MOVQ 16(SP), BX; MOVQ BX, PTRACE_RIP(AX)
|
|
MOVQ 24(SP), CX; MOVQ CX, PTRACE_CS(AX)
|
|
MOVQ 32(SP), DX; MOVQ DX, PTRACE_FLAGS(AX)
|
|
MOVQ 40(SP), DI; MOVQ DI, PTRACE_RSP(AX)
|
|
MOVQ 48(SP), SI; MOVQ SI, PTRACE_SS(AX)
|
|
|
|
CALL ·jumpToUser(SB)
|
|
|
|
// Restore kernel FS_BASE.
|
|
MOVQ ENTRY_CPU_SELF(GS), AX // Load vCPU.
|
|
MOVQ CPU_REGISTERS+PTRACE_FS_BASE(AX), BX
|
|
|
|
PUSHQ BX // First argument (FS_BASE)
|
|
CALL ·writeFS(SB)
|
|
POPQ BX
|
|
|
|
// Copy out and return.
|
|
MOVQ ENTRY_CPU_SELF(GS), AX // Load vCPU.
|
|
MOVQ 0(SP), BX // Load vector.
|
|
MOVQ 8(SP), CX // Load error code.
|
|
MOVQ CPU_REGISTERS+PTRACE_RSP(AX), SP // Original stack (kernel version).
|
|
MOVQ CPU_REGISTERS+PTRACE_RBP(AX), BP // Original base pointer.
|
|
MOVQ CX, CPU_ERROR_CODE(AX) // Set error code.
|
|
MOVQ $1, CPU_ERROR_TYPE(AX) // Set error type to user.
|
|
MOVQ BX, 32(SP) // Output vector.
|
|
RET
|
|
|
|
kernel:
|
|
// As per above, we can save directly.
|
|
PUSHQ AX
|
|
MOVQ ENTRY_CPU_SELF(GS), AX // Load vCPU.
|
|
REGISTERS_SAVE(AX, CPU_REGISTERS)
|
|
POPQ BX
|
|
MOVQ BX, CPU_REGISTERS+PTRACE_RAX(AX)
|
|
MOVQ BX, CPU_REGISTERS+PTRACE_ORIGRAX(AX)
|
|
MOVQ 16(SP), BX; MOVQ BX, CPU_REGISTERS+PTRACE_RIP(AX)
|
|
MOVQ 32(SP), BX; MOVQ BX, CPU_REGISTERS+PTRACE_FLAGS(AX)
|
|
MOVQ 40(SP), BX; MOVQ BX, CPU_REGISTERS+PTRACE_RSP(AX)
|
|
|
|
// Set the error code and adjust the stack.
|
|
MOVQ 8(SP), BX // Load the error code.
|
|
MOVQ BX, CPU_ERROR_CODE(AX) // Copy out to the CPU.
|
|
MOVQ 0(SP), BX // Load the error code.
|
|
MOVQ BX, CPU_VECTOR(AX) // Copy out to the CPU.
|
|
BYTE $0x0f; BYTE $0x20; BYTE $0xd3; // MOV CR2, RBX
|
|
MOVQ BX, CPU_FAULT_ADDR(AX)
|
|
MOVQ $0, CPU_ERROR_TYPE(AX) // Set error type to kernel.
|
|
|
|
// Save floating point state. CPU.floatingPointState is a slice, so the
|
|
// first word of CPU.floatingPointState is a pointer to the destination
|
|
// array.
|
|
MOVQ CPU_FPU_STATE(AX), DI
|
|
MOVB CPU_HAS_XSAVE(AX), BX
|
|
MOVB CPU_HAS_XSAVEOPT(AX), CX
|
|
TESTB BX, BX
|
|
JZ no_xsave
|
|
// Use xsave/xsaveopt to save all extended state.
|
|
MOVL $XCR0_EAX, AX
|
|
MOVL $XCR0_EDX, DX
|
|
TESTB CX, CX
|
|
JZ no_xsaveopt
|
|
BYTE $0x48; BYTE $0x0f; BYTE $0xae; BYTE $0x37; // XSAVEOPT64 0(DI)
|
|
JMP fpsave_done
|
|
no_xsaveopt:
|
|
BYTE $0x48; BYTE $0x0f; BYTE $0xae; BYTE $0x27; // XSAVE64 0(DI)
|
|
JMP fpsave_done
|
|
no_xsave:
|
|
FXSAVE64 0(DI)
|
|
fpsave_done:
|
|
|
|
// Call the exception trampoline.
|
|
MOVQ 0(SP), BX // BX contains the vector.
|
|
LOAD_KERNEL_STACK(GS)
|
|
MOVQ ENTRY_CPU_SELF(GS), AX // AX contains the vCPU.
|
|
PUSHQ BX // Second argument (vector).
|
|
PUSHQ AX // First argument (vCPU).
|
|
CALL ·jumpToUser(SB)
|
|
CALL ·kernelException(SB) // Call the trampoline.
|
|
POPQ BX // Pop vector.
|
|
POPQ AX // Pop vCPU.
|
|
|
|
// We only trigger a bluepill entry in the bluepill function, and can
|
|
// therefore be guaranteed that there is no floating point state to be
|
|
// loaded on resuming from halt.
|
|
JMP ·resume(SB)
|
|
|
|
#define EXCEPTION_WITH_ERROR(value, symbol, addr) \
|
|
ADDR_OF_FUNC(addr, symbol); \
|
|
TEXT symbol,NOSPLIT|NOFRAME,$0; \
|
|
PUSHQ $value; \
|
|
JMP ·exception(SB);
|
|
|
|
#define EXCEPTION_WITHOUT_ERROR(value, symbol, addr) \
|
|
ADDR_OF_FUNC(addr, symbol); \
|
|
TEXT symbol,NOSPLIT|NOFRAME,$0; \
|
|
PUSHQ $0x0; \
|
|
PUSHQ $value; \
|
|
JMP ·exception(SB);
|
|
|
|
EXCEPTION_WITHOUT_ERROR(DivideByZero, ·divideByZero(SB), ·addrOfDivideByZero(SB))
|
|
EXCEPTION_WITHOUT_ERROR(Debug, ·debug(SB), ·addrOfDebug(SB))
|
|
EXCEPTION_WITHOUT_ERROR(NMI, ·nmi(SB), ·addrOfNMI(SB))
|
|
EXCEPTION_WITHOUT_ERROR(Breakpoint, ·breakpoint(SB), ·addrOfBreakpoint(SB))
|
|
EXCEPTION_WITHOUT_ERROR(Overflow, ·overflow(SB), ·addrOfOverflow(SB))
|
|
EXCEPTION_WITHOUT_ERROR(BoundRangeExceeded, ·boundRangeExceeded(SB), ·addrOfBoundRangeExceeded(SB))
|
|
EXCEPTION_WITHOUT_ERROR(InvalidOpcode, ·invalidOpcode(SB), ·addrOfInvalidOpcode(SB))
|
|
EXCEPTION_WITHOUT_ERROR(DeviceNotAvailable, ·deviceNotAvailable(SB), ·addrOfDeviceNotAvailable(SB))
|
|
EXCEPTION_WITH_ERROR(DoubleFault, ·doubleFault(SB), ·addrOfDoubleFault(SB))
|
|
EXCEPTION_WITHOUT_ERROR(CoprocessorSegmentOverrun, ·coprocessorSegmentOverrun(SB), ·addrOfCoprocessorSegmentOverrun(SB))
|
|
EXCEPTION_WITH_ERROR(InvalidTSS, ·invalidTSS(SB), ·addrOfInvalidTSS(SB))
|
|
EXCEPTION_WITH_ERROR(SegmentNotPresent, ·segmentNotPresent(SB), ·addrOfSegmentNotPresent(SB))
|
|
EXCEPTION_WITH_ERROR(StackSegmentFault, ·stackSegmentFault(SB), ·addrOfStackSegmentFault(SB))
|
|
EXCEPTION_WITH_ERROR(GeneralProtectionFault, ·generalProtectionFault(SB), ·addrOfGeneralProtectionFault(SB))
|
|
EXCEPTION_WITH_ERROR(PageFault, ·pageFault(SB), ·addrOfPageFault(SB))
|
|
EXCEPTION_WITHOUT_ERROR(X87FloatingPointException, ·x87FloatingPointException(SB), ·addrOfX87FloatingPointException(SB))
|
|
EXCEPTION_WITH_ERROR(AlignmentCheck, ·alignmentCheck(SB), ·addrOfAlignmentCheck(SB))
|
|
EXCEPTION_WITHOUT_ERROR(MachineCheck, ·machineCheck(SB), ·addrOfMachineCheck(SB))
|
|
EXCEPTION_WITHOUT_ERROR(SIMDFloatingPointException, ·simdFloatingPointException(SB), ·addrOfSimdFloatingPointException(SB))
|
|
EXCEPTION_WITHOUT_ERROR(VirtualizationException, ·virtualizationException(SB), ·addrOfVirtualizationException(SB))
|
|
EXCEPTION_WITH_ERROR(SecurityException, ·securityException(SB), ·addrOfSecurityException(SB))
|
|
EXCEPTION_WITHOUT_ERROR(SyscallInt80, ·syscallInt80(SB), ·addrOfSyscallInt80(SB))
|