Add 64K page size support for ARM64

This adds support for running gVisor on ARM64 systems with 64K page size (e.g., NVIDIA Grace CPU). Key changes: 1. hostarch: Split page size constants into separate files selected by the `pagesize_64k` build tag. 64K pages use PageShift=16 and HugePageShift=29 (512MB huge pages). Remove uint16 from the bytecount interface since 64K (2^16) cannot be represented by uint16. 2. ring0: Move page table layout constants to page-size-specific files. 64K pages use larger page table entries (64K per table) and skip the PUD level. 3. ring0/pagetables: Add 3-level page table walker for 64K pages (PGD→PMD→PTE, no PUD). Each level indexes 8192 entries (13 bits) with 512MB sect pages at the PMD level. 4. sentry/arch: Fix mmap layout constants that overflowed the 48-bit address space with 64K pages. Use fixed values for maxMmapRand64 (1<<45 = 32TB) and minMmapRand64 (1<<30 = 1GB), and compute maxStackRand64 using hostarch.PageShift. 5. Bazel build system: Add `--define=pagesize=64k` support that propagates the `pagesize_64k` Go build tag to all binaries and tests. Build with `bazel build --define=pagesize=64k //runsc:runsc` to enable. Note: Currently only the ptrace platform is supported for 64K pages. KVM and systrap platforms require additional work. Updates #8196
2026-09-24 17:04:14 +00:00 · 2026-03-04 14:00:13 -08:00
parent 67c02b0506
commit 28cf7f346d
26 changed files with 1075 additions and 137 deletions
@@ -37,6 +37,8 @@ go_library(
        "addr_unsafe.go",
        "hostarch.go",
        "hostarch_arm64.go",
+        "hostarch_arm64_4k.go",
+        "hostarch_arm64_64k.go",
        "hostarch_x86.go",
        "memory_type.go",
        "sizes_util.go",
@@ -13,21 +13,15 @@
 // limitations under the License.

 //go:build arm64
-// +build arm64

 package hostarch

-import (
-	"encoding/binary"
-
-	"golang.org/x/sys/unix"
-)
+import "encoding/binary"

 const (
 	// PageSize is the system page size.
 	// arm64 support 4K/16K/64K page size,
 	// which can be get by unix.Getpagesize().
-	// Currently, only 4K page size is supported.
 	PageSize = 1 << PageShift

 	// HugePageSize is the system huge page size.
@@ -36,14 +30,6 @@ const (
 	// CacheLineSize is the size of the cache line.
 	CacheLineSize = 1 << CacheLineShift

-	// PageShift is the binary log of the system page size.
-	PageShift = 12
-
-	// HugePageShift is the binary log of the system huge page size.
-	// Should be calculated by "PageShift + (PageShift - 3)"
-	// when multiple page size support is ready.
-	HugePageShift = 21
-
 	// CacheLineShift is the binary log of the cache line size.
 	CacheLineShift = 6
 )
@@ -91,10 +77,3 @@ func ESRAccessType(code uint64) AccessType {
 func UntaggedUserAddr(addr Addr) Addr {
 	return Addr(int64(addr<<8) >> 8)
 }
-
-func init() {
-	// Make sure the page size is 4K on arm64 platform.
-	if size := unix.Getpagesize(); size != PageSize {
-		panic("Only 4K page size is supported on arm64!")
-	}
-}
@@ -0,0 +1,37 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build arm64 && !pagesize_64k
+
+package hostarch
+
+import "golang.org/x/sys/unix"
+
+const (
+	// PageShift is the binary log of the system page size.
+	// 4K pages: 2^12 = 4096
+	PageShift = 12
+
+	// HugePageShift is the binary log of the system huge page size.
+	// For 4K pages: PageShift + (PageShift - 3) = 12 + 9 = 21
+	// This gives 2MB huge pages.
+	HugePageShift = 21
+)
+
+func init() {
+	// Make sure the page size is 4K on arm64 platform.
+	if size := unix.Getpagesize(); size != PageSize {
+		panic("Only 4K page size is supported on arm64!")
+	}
+}
@@ -0,0 +1,37 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build arm64 && pagesize_64k
+
+package hostarch
+
+import "golang.org/x/sys/unix"
+
+const (
+	// PageShift is the binary log of the system page size.
+	// 64K pages: 2^16 = 65536
+	PageShift = 16
+
+	// HugePageShift is the binary log of the system huge page size.
+	// For 64K pages: PageShift + (PageShift - 3) = 16 + 13 = 29
+	// This gives 512MB huge pages.
+	HugePageShift = 29
+)
+
+func init() {
+	// Make sure the page size is 64K on arm64 platform.
+	if size := unix.Getpagesize(); size != PageSize {
+		panic("Only 64K page size is supported on arm64 with pagesize_64k build tag!")
+	}
+}
@@ -14,7 +14,7 @@ const (
 )

 type bytecount interface {
-	~uint | ~uint16 | ~uint32 | ~uint64 | ~uintptr
+	~uint | ~uint32 | ~uint64 | ~uintptr
 }

 type hugebytecount interface {
@@ -9,6 +9,8 @@ go_library(
    name = "ring0",
    srcs = [
        "aarch64.go",
+        "aarch64_4k.go",
+        "aarch64_64k.go",
        "defs.go",
        "defs_amd64.go",
        "defs_arm64.go",
@@ -13,22 +13,9 @@
 // limitations under the License.

 //go:build arm64
-// +build arm64

 package ring0

-// Useful bits.
-const (
-	_PGD_PGT_BASE = 0x1000
-	_PGD_PGT_SIZE = 0x1000
-	_PUD_PGT_BASE = 0x2000
-	_PUD_PGT_SIZE = 0x1000
-	_PMD_PGT_BASE = 0x3000
-	_PMD_PGT_SIZE = 0x4000
-	_PTE_PGT_BASE = 0x7000
-	_PTE_PGT_SIZE = 0x1000
-)
-
 const (
 	// VirtualAddressBits is fixed at 48.
 	VirtualAddressBits = 48
@@ -0,0 +1,29 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build arm64 && !pagesize_64k
+
+package ring0
+
+// Page table layout constants for 4K page size.
+const (
+	_PGD_PGT_BASE = 0x1000
+	_PGD_PGT_SIZE = 0x1000
+	_PUD_PGT_BASE = 0x2000
+	_PUD_PGT_SIZE = 0x1000
+	_PMD_PGT_BASE = 0x3000
+	_PMD_PGT_SIZE = 0x4000
+	_PTE_PGT_BASE = 0x7000
+	_PTE_PGT_SIZE = 0x1000
+)
@@ -0,0 +1,36 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build arm64 && pagesize_64k
+
+package ring0
+
+// Page table layout constants for 64K page size.
+//
+// With 64K pages, we use a 3-level page table structure:
+// - PGD (Page Global Directory)
+// - PMD (Page Middle Directory)
+// - PTE (Page Table Entry)
+//
+// Each page table has 8192 entries (64K / 8 bytes per entry).
+// For 48-bit VA, PGD only uses 64 entries (6 bits), but we allocate full page.
+const (
+	_PGD_PGT_BASE = 0x10000
+	_PGD_PGT_SIZE = 0x10000 // 64K for PGD page table
+	// No PUD level in 64K page mode
+	_PMD_PGT_BASE = 0x20000
+	_PMD_PGT_SIZE = 0x40000 // 256K = 4 page tables for initial kernel mapping
+	_PTE_PGT_BASE = 0x60000
+	_PTE_PGT_SIZE = 0x10000 // 64K for PTE page table
+)
@@ -21,7 +21,7 @@ package(
        ],
        visibility = [":__pkg__"],
    )
-    for arch in ("amd64", "arm64")
+    for arch in ("amd64", "arm64", "arm64_64k")
 ]

 [
@@ -37,7 +37,7 @@ package(
        },
    )
    for op in ("map", "unmap", "lookup", "empty", "check")
-    for arch in ("amd64", "arm64")
+    for arch in ("amd64", "arm64", "arm64_64k")
 ]

 go_library(
@@ -49,6 +49,8 @@ go_library(
        "pagetables_aarch64.go",
        "pagetables_amd64.go",
        "pagetables_arm64.go",
+        "pagetables_arm64_4k.go",
+        "pagetables_arm64_64k.go",
        "pagetables_x86.go",
        "pcids.go",
        "pcids_aarch64.go",
@@ -56,15 +58,20 @@ go_library(
        "pcids_x86.go",
        "walker_amd64.go",
        "walker_arm64.go",
+        "walker_arm64_64k.go",
        "walker_generic.go",
        ":walker_empty_amd64",
        ":walker_empty_arm64",
+        ":walker_empty_arm64_64k",
        ":walker_lookup_amd64",
        ":walker_lookup_arm64",
+        ":walker_lookup_arm64_64k",
        ":walker_map_amd64",
        ":walker_map_arm64",
+        ":walker_map_arm64_64k",
        ":walker_unmap_amd64",
        ":walker_unmap_arm64",
+        ":walker_unmap_arm64_64k",
    ],
    visibility = [
        "//pkg/ring0:__subpackages__",
@@ -83,9 +90,12 @@ go_test(
    srcs = [
        "pagetables_amd64_test.go",
        "pagetables_arm64_test.go",
+        "pagetables_arm64_64k_test.go",
+        "pagetables_common_test.go",
        "pagetables_test.go",
        ":walker_check_amd64",
        ":walker_check_arm64",
+        ":walker_check_arm64_64k",
    ],
    library = ":pagetables",
    deps = [
@@ -12,34 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

+//go:build arm64
+
 package pagetables

-// Address constraints.
-//
-// The lowerTop and upperBottom currently apply to four-level pagetables;
-// additional refactoring would be necessary to support five-level pagetables.
+// Address space layout constants shared by all ARM64 page sizes.
 const (
 	lowerTop    = 0x0000ffffffffffff
 	upperBottom = 0xffff000000000000
-	pteShift    = 12
-	pmdShift    = 21
-	pudShift    = 30
-	pgdShift    = 39
-
-	pteMask = 0x1ff << pteShift
-	pmdMask = 0x1ff << pmdShift
-	pudMask = 0x1ff << pudShift
-	pgdMask = 0x1ff << pgdShift
-
-	pteSize = 1 << pteShift
-	pmdSize = 1 << pmdShift
-	pudSize = 1 << pudShift
-	pgdSize = 1 << pgdShift

 	ttbrASIDOffset = 48
 	ttbrASIDMask   = 0xff
-
-	entriesPerPage = 512
 )

 // InitArch does some additional initialization related to the architecture.
@@ -67,6 +50,3 @@ func (p *PageTables) cloneUpperShared() {
 	p.archPageTables.root = p.upperSharedPageTables.archPageTables.root
 	p.archPageTables.rootPhysical = p.upperSharedPageTables.archPageTables.rootPhysical
 }
-
-// PTEs is a collection of entries.
-type PTEs [entriesPerPage]PTE
@@ -0,0 +1,40 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build arm64 && !pagesize_64k
+
+package pagetables
+
+// Page table constants for 4K page size (4-level page tables).
+const (
+	pteShift = 12
+	pmdShift = 21
+	pudShift = 30
+	pgdShift = 39
+
+	pteMask = 0x1ff << pteShift
+	pmdMask = 0x1ff << pmdShift
+	pudMask = 0x1ff << pudShift
+	pgdMask = 0x1ff << pgdShift
+
+	pteSize = 1 << pteShift
+	pmdSize = 1 << pmdShift
+	pudSize = 1 << pudShift
+	pgdSize = 1 << pgdShift
+
+	entriesPerPage = 512
+)
+
+// PTEs is a collection of entries.
+type PTEs [entriesPerPage]PTE
@@ -0,0 +1,53 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build arm64 && pagesize_64k
+
+package pagetables
+
+// Page table constants for 64K page size (3-level page tables).
+//
+// With 64K pages, we use a 3-level page table structure:
+// - PGD (Page Global Directory)
+// - PMD (Page Middle Directory)
+// - PTE (Page Table Entry)
+//
+// Each page table page contains 8192 entries (64K / 8 bytes).
+// Each level uses 13 bits for indexing.
+//
+// Virtual address layout (48-bit):
+// [47:42] - PGD index (6 bits, 64 entries at top level)
+// [41:29] - PMD index (13 bits, 8192 entries)
+// [28:16] - PTE index (13 bits, 8192 entries)
+// [15:0]  - Page offset (16 bits, 64K page)
+const (
+	pteShift = 16
+	pmdShift = 29
+	pgdShift = 42
+
+	// Mask for 13-bit index (8192 entries)
+	pteMask = 0x1fff << pteShift
+	pmdMask = 0x1fff << pmdShift
+	// Top level only has 6 bits (64 entries) for 48-bit VA
+	pgdMask = 0x3f << pgdShift
+
+	pteSize = 1 << pteShift // 64K
+	pmdSize = 1 << pmdShift // 512M
+	pgdSize = 1 << pgdShift // 4T
+
+	entriesPerPage = 8192
+)
+
+// PTEs is a collection of entries.
+type PTEs [entriesPerPage]PTE
@@ -0,0 +1,64 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build arm64 && pagesize_64k
+// +build arm64,pagesize_64k
+
+package pagetables
+
+import (
+	"testing"
+
+	"gvisor.dev/gvisor/pkg/hostarch"
+)
+
+// Test512MAnd64K tests mapping of 512MB sect pages and 64K pages.
+func Test512MAnd64K(t *testing.T) {
+	pt := New(NewRuntimeAllocator())
+
+	// Map a small page (64K) and a huge page (512MB).
+	pt.Map(0x400000, pteSize, MapOpts{AccessType: hostarch.ReadWrite, User: true}, pteSize*42)
+	pt.Map(0x0000ff0000000000, pmdSize, MapOpts{AccessType: hostarch.Read, User: true}, pmdSize*47)
+
+	pt.Map(0xffff000000400000, pteSize, MapOpts{AccessType: hostarch.ReadWrite, User: false}, pteSize*42)
+	pt.Map(0xffffff0000000000, pmdSize, MapOpts{AccessType: hostarch.Read, User: false}, pmdSize*47)
+
+	checkMappings(t, pt, []mapping{
+		{0x400000, pteSize, pteSize * 42, MapOpts{AccessType: hostarch.ReadWrite, User: true}},
+		{0x0000ff0000000000, pmdSize, pmdSize * 47, MapOpts{AccessType: hostarch.Read, User: true}},
+		{0xffff000000400000, pteSize, pteSize * 42, MapOpts{AccessType: hostarch.ReadWrite, User: false}},
+		{0xffffff0000000000, pmdSize, pmdSize * 47, MapOpts{AccessType: hostarch.Read, User: false}},
+	})
+}
+
+// TestSplit512MPage tests splitting a 512MB sect page.
+func TestSplit512MPage(t *testing.T) {
+	pt := New(NewRuntimeAllocator())
+
+	// Map a sect page (512MB) and knock out the middle.
+	pt.Map(0x0000ff0000000000, pmdSize, MapOpts{AccessType: hostarch.Read, User: true}, pmdSize*42)
+	pt.Unmap(hostarch.Addr(0x0000ff0000000000+pteSize), pmdSize-(2*pteSize))
+
+	checkMappings(t, pt, []mapping{
+		{0x0000ff0000000000, pteSize, pmdSize * 42, MapOpts{AccessType: hostarch.Read, User: true}},
+		{0x0000ff0000000000 + pmdSize - pteSize, pteSize, pmdSize*42 + pmdSize - pteSize, MapOpts{AccessType: hostarch.Read, User: true}},
+	})
+}
+
+func TestNumMemoryTypes(t *testing.T) {
+	// MAIR accommodates up to 8 entries.
+	if hostarch.NumMemoryTypes > 8 {
+		t.Errorf("PTE.Set() and PTE.Opts() must be altered to map %d MemoryTypes to a smaller set of MAIR entries", hostarch.NumMemoryTypes)
+	}
+}
@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-//go:build arm64
-// +build arm64
+//go:build arm64 && !pagesize_64k
+// +build arm64,!pagesize_64k

 package pagetables

@@ -0,0 +1,85 @@
+// Copyright 2018 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pagetables
+
+import (
+	"testing"
+)
+
+type mapping struct {
+	start  uintptr
+	length uintptr
+	addr   uintptr
+	opts   MapOpts
+}
+
+type checkVisitor struct {
+	expected []mapping // Input.
+	current  int       // Temporary.
+	found    []mapping // Output.
+	failed   string    // Output.
+}
+
+func (v *checkVisitor) visit(start uintptr, pte *PTE, align uintptr) bool {
+	v.found = append(v.found, mapping{
+		start:  start,
+		length: align + 1,
+		addr:   pte.Address(),
+		opts:   pte.Opts(),
+	})
+	if v.failed != "" {
+		// Don't keep looking for errors.
+		return false
+	}
+
+	if v.current >= len(v.expected) {
+		v.failed = "more mappings than expected"
+	} else if v.expected[v.current].start != start {
+		v.failed = "start didn't match expected"
+	} else if v.expected[v.current].length != (align + 1) {
+		v.failed = "end didn't match expected"
+	} else if v.expected[v.current].addr != pte.Address() {
+		v.failed = "address didn't match expected"
+	} else if v.expected[v.current].opts != pte.Opts() {
+		v.failed = "opts didn't match"
+	}
+	v.current++
+	return true
+}
+
+func (*checkVisitor) requiresAlloc() bool { return false }
+
+func (*checkVisitor) requiresSplit() bool { return false }
+
+func checkMappings(t *testing.T, pt *PageTables, m []mapping) {
+	// Iterate over all the mappings.
+	w := checkWalker{
+		pageTables: pt,
+		visitor: checkVisitor{
+			expected: m,
+		},
+	}
+	w.iterateRange(0, ^uintptr(0))
+
+	// Were we expected additional mappings?
+	if w.visitor.failed == "" && w.visitor.current != len(w.visitor.expected) {
+		w.visitor.failed = "insufficient mappings found"
+	}
+
+	// Emit a meaningful error message on failure.
+	if w.visitor.failed != "" {
+		t.Errorf("%s; got %#v, wanted %#v", w.visitor.failed, w.visitor.found, w.visitor.expected)
+	}
+}
@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

+//go:build !pagesize_64k
+
 package pagetables

 import (
@@ -20,72 +22,6 @@ import (
 	"gvisor.dev/gvisor/pkg/hostarch"
 )

-type mapping struct {
-	start  uintptr
-	length uintptr
-	addr   uintptr
-	opts   MapOpts
-}
-
-type checkVisitor struct {
-	expected []mapping // Input.
-	current  int       // Temporary.
-	found    []mapping // Output.
-	failed   string    // Output.
-}
-
-func (v *checkVisitor) visit(start uintptr, pte *PTE, align uintptr) bool {
-	v.found = append(v.found, mapping{
-		start:  start,
-		length: align + 1,
-		addr:   pte.Address(),
-		opts:   pte.Opts(),
-	})
-	if v.failed != "" {
-		// Don't keep looking for errors.
-		return false
-	}
-
-	if v.current >= len(v.expected) {
-		v.failed = "more mappings than expected"
-	} else if v.expected[v.current].start != start {
-		v.failed = "start didn't match expected"
-	} else if v.expected[v.current].length != (align + 1) {
-		v.failed = "end didn't match expected"
-	} else if v.expected[v.current].addr != pte.Address() {
-		v.failed = "address didn't match expected"
-	} else if v.expected[v.current].opts != pte.Opts() {
-		v.failed = "opts didn't match"
-	}
-	v.current++
-	return true
-}
-
-func (*checkVisitor) requiresAlloc() bool { return false }
-
-func (*checkVisitor) requiresSplit() bool { return false }
-
-func checkMappings(t *testing.T, pt *PageTables, m []mapping) {
-	// Iterate over all the mappings.
-	w := checkWalker{
-		pageTables: pt,
-		visitor: checkVisitor{
-			expected: m,
-		},
-	}
-	w.iterateRange(0, ^uintptr(0))
-
-	// Were we expected additional mappings?
-	if w.visitor.failed == "" && w.visitor.current != len(w.visitor.expected) {
-		w.visitor.failed = "insufficient mappings found"
-	}
-
-	// Emit a meaningful error message on failure.
-	if w.visitor.failed != "" {
-		t.Errorf("%s; got %#v, wanted %#v", w.visitor.failed, w.visitor.found, w.visitor.expected)
-	}
-}
-
 func TestUnmap(t *testing.T) {
 	pt := New(NewRuntimeAllocator())

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-//go:build arm64
-// +build arm64
+//go:build arm64 && !pagesize_64k
+// +build arm64,!pagesize_64k

 package pagetables

@@ -0,0 +1,160 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build arm64 && pagesize_64k
+// +build arm64,pagesize_64k
+
+package pagetables
+
+// iterateRangeCanonical walks a canonical range.
+//
+// This is a 3-level page table walker for 64K pages:
+// PGD -> PMD -> PTE (no PUD level)
+//
+//go:nosplit
+func (w *Walker) iterateRangeCanonical(start, end uintptr) bool {
+	pgdEntryIndex := w.pageTables.root
+	if start >= upperBottom {
+		pgdEntryIndex = w.pageTables.archPageTables.root
+	}
+
+	for pgdIndex := (uint16((start & pgdMask) >> pgdShift)); start < end && pgdIndex < entriesPerPage; pgdIndex++ {
+		var (
+			pgdEntry   = &pgdEntryIndex[pgdIndex]
+			pmdEntries *PTEs
+		)
+		if !pgdEntry.Valid() {
+			if !w.visitor.requiresAlloc() {
+				// Skip over this entry.
+				start = next(start, pgdSize)
+				continue
+			}
+
+			// Allocate a new pgd.
+			pmdEntries = w.pageTables.Allocator.NewPTEs()
+			pgdEntry.setPageTable(w.pageTables, pmdEntries)
+		} else {
+			pmdEntries = w.pageTables.Allocator.LookupPTEs(pgdEntry.Address())
+		}
+
+		// Map the next level (PMD -> PTE, no PUD in 64K page mode).
+		clearPMDEntries := uint16(0)
+
+		for pmdIndex := uint16((start & pmdMask) >> pmdShift); start < end && pmdIndex < entriesPerPage; pmdIndex++ {
+			var (
+				pmdEntry   = &pmdEntries[pmdIndex]
+				pteEntries *PTEs
+			)
+			if !pmdEntry.Valid() {
+				if !w.visitor.requiresAlloc() {
+					// Skip over this entry.
+					clearPMDEntries++
+					start = next(start, pmdSize)
+					continue
+				}
+
+				// This level has 512-MB sect pages (for 64K page size).
+				// If this region is contained in a single PMD entry,
+				// we can skip allocating a new page.
+				if start&(pmdSize-1) == 0 && end-start >= pmdSize {
+					pmdEntry.SetSect()
+					if !w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1) {
+						return false
+					}
+					if pmdEntry.Valid() {
+						start = next(start, pmdSize)
+						continue
+					}
+				}
+
+				// Allocate a new pmd.
+				pteEntries = w.pageTables.Allocator.NewPTEs()
+				pmdEntry.setPageTable(w.pageTables, pteEntries)
+
+			} else if pmdEntry.IsSect() {
+				// Does this page need to be split?
+				if w.visitor.requiresSplit() && (start&(pmdSize-1) != 0 || end < next(start, pmdSize)) {
+					// Install the relevant entries.
+					pteEntries = w.pageTables.Allocator.NewPTEs()
+					for index := uint16(0); index < entriesPerPage; index++ {
+						pteEntries[index].Set(
+							pmdEntry.Address()+(pteSize*uintptr(index)),
+							pmdEntry.Opts())
+					}
+					pmdEntry.setPageTable(w.pageTables, pteEntries)
+				} else {
+					// A sect page to be checked directly.
+					if !w.visitor.visit(uintptr(start), pmdEntry, pmdSize-1) {
+						return false
+					}
+
+					// Might have been cleared.
+					if !pmdEntry.Valid() {
+						clearPMDEntries++
+					}
+
+					// Note that the sect page was changed.
+					start = next(start, pmdSize)
+					continue
+				}
+
+			} else {
+				pteEntries = w.pageTables.Allocator.LookupPTEs(pmdEntry.Address())
+			}
+
+			// Map the next level, since this is valid.
+			clearPTEEntries := uint16(0)
+
+			for pteIndex := uint16((start & pteMask) >> pteShift); start < end && pteIndex < entriesPerPage; pteIndex++ {
+				var (
+					pteEntry = &pteEntries[pteIndex]
+				)
+				if !pteEntry.Valid() && !w.visitor.requiresAlloc() {
+					clearPTEEntries++
+					start += pteSize
+					continue
+				}
+
+				// At this point, we are guaranteed that start%pteSize == 0.
+				if !w.visitor.visit(uintptr(start), pteEntry, pteSize-1) {
+					return false
+				}
+				if !pteEntry.Valid() {
+					if w.visitor.requiresAlloc() {
+						panic("PTE not set after iteration with requiresAlloc!")
+					}
+					clearPTEEntries++
+				}
+
+				// Note that the pte was changed.
+				start += pteSize
+				continue
+			}
+
+			// Check if we no longer need this page.
+			if clearPTEEntries == entriesPerPage {
+				pmdEntry.Clear()
+				w.pageTables.Allocator.FreePTEs(pteEntries)
+				clearPMDEntries++
+			}
+		}
+
+		// Check if we no longer need this page.
+		if clearPMDEntries == entriesPerPage {
+			pgdEntry.Clear()
+			w.pageTables.Allocator.FreePTEs(pmdEntries)
+		}
+	}
+	return true
+}
@@ -40,11 +40,14 @@ const (

 	// maxStackRand64 is the maximum randomization to apply to the stack.
 	// It is defined by arch/arm64/mm/mmap.c:(STACK_RND_MASK << PAGE_SHIFT) in Linux.
-	maxStackRand64 = 0x3ffff << 12 // 16 GB
+	maxStackRand64 = 0x3ffff << hostarch.PageShift

 	// maxMmapRand64 is the maximum randomization to apply to the mmap
 	// layout. It is defined by arch/arm64/mm/mmap.c:arch_mmap_rnd in Linux.
-	maxMmapRand64 = (1 << 33) * hostarch.PageSize
+	// For 4K pages (PageShift=12): 1 << 45 = 32TB
+	// For 64K pages (PageShift=16): 1 << 45 = 32TB (same)
+	// We use a fixed value to avoid exceeding the 48-bit address space.
+	maxMmapRand64 = 1 << 45

 	// minGap64 is the minimum gap to leave at the top of the address space
 	// for the stack. It is defined by arch/arm64/mm/mmap.c:MIN_GAP in Linux.
@@ -72,7 +75,8 @@ const (

 	// minMmapRand64 is the smallest we are willing to make the
 	// randomization to stay above preferredTopDownBaseMin.
-	minMmapRand64 = (1 << 18) * hostarch.PageSize
+	// Use a fixed value (1GB) to be consistent across page sizes.
+	minMmapRand64 = 1 << 30
 )

 // Context64 represents an ARM64 context.
@@ -203,7 +203,10 @@ go_library(
 go_test(
    name = "pgalloc_test",
    size = "small",
-    srcs = ["pgalloc_test.go"],
+    srcs = [
+        "pgalloc_64k_test.go",
+        "pgalloc_test.go",
+    ],
    library = ":pgalloc",
    deps = [
        "//pkg/hostarch",
@@ -0,0 +1,468 @@
+// Copyright 2018 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build arm64 && pagesize_64k
+
+package pgalloc
+
+import (
+	"testing"
+
+	"gvisor.dev/gvisor/pkg/hostarch"
+	"gvisor.dev/gvisor/pkg/sentry/memmap"
+)
+
+const (
+	page     = hostarch.PageSize
+	hugepage = hostarch.HugePageSize
+)
+
+// existingSegment represents a range of pages in a test MemoryFile that is not
+// void or free.
+type existingSegment struct {
+	start uint64
+	end   uint64
+	state int
+}
+
+// Possible values for existingSegment.state:
+const (
+	existingUnspecified = iota
+	existingUsed
+	existingWaste
+	existingReleasing // or sub-releasing
+)
+
+// TestFindAllocatable64K tests allocation for 64K page size.
+// Note: For 64K pages, hugepage is 512MB and chunkSize is 1GB,
+// so chunkSize only holds 2 hugepages. Tests are adjusted accordingly.
+func TestFindAllocatable64K(t *testing.T) {
+	for _, test := range []struct {
+		name string
+		// Initial state:
+		chunkHuge []bool
+		existing  []existingSegment
+		// Allocation parameters:
+		length  uint64
+		huge    bool
+		recycle bool
+		dir     Direction
+		// Expected outcome:
+		want uint64
+	}{
+		{
+			name:   "initial small allocation, bottom-up",
+			length: page,
+			want:   0,
+		},
+		{
+			name:   "initial small allocation, top-down",
+			length: page,
+			dir:    TopDown,
+			want:   chunkSize - page,
+		},
+		{
+			name:   "initial small allocation, multiple pages, top-down",
+			length: 2 * page,
+			dir:    TopDown,
+			want:   chunkSize - 2*page,
+		},
+		{
+			name:    "initial small allocation, recycling enabled, bottom-up",
+			length:  page,
+			recycle: true,
+			want:    0,
+		},
+		{
+			name:   "initial huge allocation, bottom-up",
+			length: hugepage,
+			huge:   true,
+			want:   0,
+		},
+		{
+			name:   "initial huge allocation, top-down",
+			length: hugepage,
+			huge:   true,
+			dir:    TopDown,
+			want:   chunkSize - hugepage,
+		},
+		{
+			name:    "initial huge allocation, recycling enabled, bottom-up",
+			length:  hugepage,
+			huge:    true,
+			recycle: true,
+			want:    0,
+		},
+		{
+			name:      "huge allocation uses huge pages in new chunk",
+			chunkHuge: []bool{false},
+			length:    hugepage,
+			huge:      true,
+			want:      chunkSize,
+		},
+		{
+			name:      "huge allocation uses huge pages in existing chunk",
+			chunkHuge: []bool{false, true},
+			length:    hugepage,
+			huge:      true,
+			want:      chunkSize,
+		},
+		{
+			name:      "hugepage-sized non-huge allocation uses small pages in new chunk",
+			chunkHuge: []bool{true},
+			length:    hugepage,
+			want:      chunkSize,
+		},
+		{
+			name:      "hugepage-sized non-huge allocation uses small pages in existing chunk",
+			chunkHuge: []bool{true, false},
+			length:    hugepage,
+			want:      chunkSize,
+		},
+		{
+			name:      "bottom-up small allocation begins at start of file",
+			chunkHuge: []bool{false},
+			existing: []existingSegment{
+				{page, 2 * page, existingUsed},
+			},
+			length: page,
+			want:   0,
+		},
+		{
+			name:      "top-down small allocation begins at end of last chunk",
+			chunkHuge: []bool{false},
+			existing: []existingSegment{
+				{chunkSize - 2*page, chunkSize - page, existingUsed},
+			},
+			length: page,
+			dir:    TopDown,
+			want:   chunkSize - page,
+		},
+		{
+			name:      "bottom-up huge allocation begins at start of file",
+			chunkHuge: []bool{true},
+			existing: []existingSegment{
+				{hugepage, chunkSize, existingUsed},
+			},
+			length: hugepage,
+			huge:   true,
+			want:   0,
+		},
+		{
+			name:      "top-down huge allocation begins at end of last chunk",
+			chunkHuge: []bool{true},
+			existing: []existingSegment{
+				{0, hugepage, existingUsed},
+			},
+			length: hugepage,
+			huge:   true,
+			dir:    TopDown,
+			want:   chunkSize - hugepage,
+		},
+		{
+			name:      "bottom-up small allocation can extend multiple chunks",
+			chunkHuge: []bool{false},
+			existing: []existingSegment{
+				{chunkSize/2 - page, chunkSize / 2, existingUsed},
+			},
+			length: 2*chunkSize + page,
+			want:   chunkSize / 2,
+		},
+		{
+			name:      "top-down small allocation can extend multiple chunks",
+			chunkHuge: []bool{false},
+			existing: []existingSegment{
+				{chunkSize/2 - page, chunkSize / 2, existingUsed},
+			},
+			length: 2*chunkSize + page,
+			dir:    TopDown,
+			want:   chunkSize - page,
+		},
+		{
+			name:      "bottom-up small allocation finds first free gap",
+			chunkHuge: []bool{false},
+			existing: []existingSegment{
+				{0, page, existingUsed},
+				{2 * page, 3 * page, existingUsed},
+			},
+			length: page,
+			want:   page,
+		},
+		{
+			name:      "top-down small allocation finds last free gap",
+			chunkHuge: []bool{false},
+			existing: []existingSegment{
+				{chunkSize - page, chunkSize, existingUsed},
+				{chunkSize - 3*page, chunkSize - 2*page, existingUsed},
+			},
+			length: page,
+			dir:    TopDown,
+			want:   chunkSize - 2*page,
+		},
+		{
+			name:      "bottom-up huge allocation finds first free gap",
+			chunkHuge: []bool{true, true}, // Need 2 chunks for this test
+			existing: []existingSegment{
+				{0, hugepage, existingUsed},
+			},
+			length: hugepage,
+			huge:   true,
+			want:   hugepage,
+		},
+		{
+			name:      "bottom-up small allocation skips undersized free gap",
+			chunkHuge: []bool{false},
+			existing: []existingSegment{
+				{0, page, existingUsed},
+				{2 * page, 3 * page, existingUsed},
+			},
+			length: 2 * page,
+			want:   3 * page,
+		},
+		{
+			name:      "top-down small allocation skips undersized free gap",
+			chunkHuge: []bool{false},
+			existing: []existingSegment{
+				{chunkSize - page, chunkSize, existingUsed},
+				{chunkSize - 3*page, chunkSize - 2*page, existingUsed},
+			},
+			length: 2 * page,
+			dir:    TopDown,
+			want:   chunkSize - 5*page,
+		},
+		{
+			name:      "recycling bottom-up small allocation skips used pages",
+			chunkHuge: []bool{false},
+			existing: []existingSegment{
+				{0, page, existingUsed},
+			},
+			length:  page,
+			recycle: true,
+			want:    page,
+		},
+		{
+			name:      "recycling top-down small allocation skips used pages",
+			chunkHuge: []bool{false},
+			existing: []existingSegment{
+				{chunkSize - page, chunkSize, existingUsed},
+			},
+			length:  page,
+			recycle: true,
+			dir:     TopDown,
+			want:    chunkSize - 2*page,
+		},
+		{
+			name:      "recycling bottom-up huge allocation skips used pages",
+			chunkHuge: []bool{true, true}, // Need 2 chunks
+			existing: []existingSegment{
+				{0, hugepage, existingUsed},
+			},
+			length:  hugepage,
+			huge:    true,
+			recycle: true,
+			want:    hugepage,
+		},
+		{
+			name:      "non-recycling bottom-up small allocation skips waste pages",
+			chunkHuge: []bool{false},
+			existing: []existingSegment{
+				{0, page, existingWaste},
+			},
+			length: page,
+			want:   page,
+		},
+		{
+			name:      "non-recycling top-down small allocation skips waste pages",
+			chunkHuge: []bool{false},
+			existing: []existingSegment{
+				{chunkSize - page, chunkSize, existingWaste},
+			},
+			length: page,
+			dir:    TopDown,
+			want:   chunkSize - 2*page,
+		},
+		{
+			name:      "non-recycling bottom-up huge allocation skips waste pages",
+			chunkHuge: []bool{true, true}, // Need 2 chunks
+			existing: []existingSegment{
+				{0, hugepage, existingWaste},
+			},
+			length: hugepage,
+			huge:   true,
+			want:   hugepage,
+		},
+		{
+			name:      "recycling bottom-up small allocation recycles waste pages",
+			chunkHuge: []bool{false},
+			existing: []existingSegment{
+				{0, page, existingWaste},
+			},
+			length:  page,
+			recycle: true,
+			want:    0,
+		},
+		{
+			name:      "recycling top-down small allocation recycles waste pages",
+			chunkHuge: []bool{false},
+			existing: []existingSegment{
+				{chunkSize - page, chunkSize, existingWaste},
+			},
+			length:  page,
+			recycle: true,
+			dir:     TopDown,
+			want:    chunkSize - page,
+		},
+		{
+			name:      "recycling bottom-up huge allocation recycles waste pages",
+			chunkHuge: []bool{true},
+			existing: []existingSegment{
+				{0, hugepage, existingWaste},
+			},
+			length:  hugepage,
+			huge:    true,
+			recycle: true,
+			want:    0,
+		},
+		{
+			name:      "non-recycling bottom-up small allocation skips releasing pages",
+			chunkHuge: []bool{false},
+			existing: []existingSegment{
+				{0, page, existingReleasing},
+			},
+			length: page,
+			want:   page,
+		},
+		{
+			name:      "non-recycling top-down small allocation skips releasing pages",
+			chunkHuge: []bool{false},
+			existing: []existingSegment{
+				{chunkSize - page, chunkSize, existingReleasing},
+			},
+			length: page,
+			dir:    TopDown,
+			want:   chunkSize - 2*page,
+		},
+		{
+			name:      "non-recycling bottom-up huge allocation skips releasing pages",
+			chunkHuge: []bool{true, true}, // Need 2 chunks
+			existing: []existingSegment{
+				{0, hugepage, existingReleasing},
+			},
+			length: hugepage,
+			huge:   true,
+			want:   hugepage,
+		},
+		{
+			name:      "recycling bottom-up small allocation skips releasing pages",
+			chunkHuge: []bool{false},
+			existing: []existingSegment{
+				{0, page, existingReleasing},
+			},
+			length:  page,
+			recycle: true,
+			want:    page,
+		},
+		{
+			name:      "recycling top-down small allocation skips releasing pages",
+			chunkHuge: []bool{false},
+			existing: []existingSegment{
+				{chunkSize - page, chunkSize, existingReleasing},
+			},
+			length:  page,
+			recycle: true,
+			dir:     TopDown,
+			want:    chunkSize - 2*page,
+		},
+		{
+			name:      "recycling bottom-up huge allocation skips releasing pages",
+			chunkHuge: []bool{true, true}, // Need 2 chunks
+			existing: []existingSegment{
+				{0, hugepage, existingReleasing},
+			},
+			length:  hugepage,
+			huge:    true,
+			recycle: true,
+			want:    hugepage,
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			// Build the fake MemoryFile.
+			f := &MemoryFile{
+				opts: MemoryFileOpts{
+					ExpectHugepages:         true,
+					DisableMemoryAccounting: true,
+				},
+			}
+			f.initFields()
+			chunks := make([]chunkInfo, len(test.chunkHuge))
+			for i, huge := range test.chunkHuge {
+				chunks[i].huge = huge
+				chunkFR := memmap.FileRange{uint64(i) * chunkSize, uint64(i+1) * chunkSize}
+				if huge {
+					f.unfreeHuge.RemoveRange(chunkFR)
+				} else {
+					f.unfreeSmall.RemoveRange(chunkFR)
+				}
+			}
+			f.chunks.Store(&chunks)
+			for _, es := range test.existing {
+				f.forEachChunk(memmap.FileRange{es.start, es.end}, func(chunk *chunkInfo, chunkFR memmap.FileRange) bool {
+					unwaste, unfree := &f.unwasteSmall, &f.unfreeSmall
+					if chunk.huge {
+						unwaste, unfree = &f.unwasteHuge, &f.unfreeHuge
+					}
+					switch es.state {
+					case existingUsed:
+						unfree.InsertRange(chunkFR, unfreeInfo{refs: 1})
+					case existingWaste:
+						unfree.InsertRange(chunkFR, unfreeInfo{refs: 0})
+						unwaste.RemoveRange(chunkFR)
+					case existingReleasing:
+						unfree.InsertRange(chunkFR, unfreeInfo{refs: 0})
+					default:
+						t.Fatalf("existingSegment %+v has unknown state", es)
+					}
+					f.memAcct.InsertRange(chunkFR, memAcctInfo{
+						wasteOrReleasing: es.state != existingUsed,
+					})
+					return true
+				})
+			}
+
+			// Perform the test allocation.
+			alloc := allocState{
+				length: test.length,
+				opts: AllocOpts{
+					Huge: test.huge,
+					Dir:  test.dir,
+				},
+				huge: test.huge,
+			}
+			if test.recycle {
+				alloc.opts.Mode = AllocateCallerIndirectCommit
+				alloc.willCommit = true
+			}
+			fr, err := f.findAllocatableAndMarkUsed(&alloc)
+			if err != nil {
+				t.Fatalf("findAllocatableAndMarkUsed(%+v): failed: %v, want: %#x\n%v", alloc, err, test.want, f)
+			}
+			if fr.Start != test.want {
+				t.Errorf("findAllocatableAndMarkUsed(%+v): got: start=%#x, want: %#x\n%v", alloc, fr.Start, test.want, f)
+			}
+			if wantEnd := test.want + test.length; fr.End != wantEnd {
+				t.Errorf("findAllocatableAndMarkUsed(%+v): got: end=%#x, want: %#x\n%v", alloc, fr.End, wantEnd, f)
+			}
+		})
+	}
+}
@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

+
+//go:build !pagesize_64k
 package pgalloc

 import (
@@ -57,3 +57,18 @@ genrule(
    ],
    visibility = ["//:sandbox"],
 )
+
+# Page size configurations for ARM64
+config_setting(
+    name = "pagesize_64k",
+    define_values = {"pagesize": "64k"},
+)
+
+# Combined config: ARM64 with 64K page size
+config_setting(
+    name = "arm64_pagesize_64k",
+    constraint_values = [
+        "@platforms//cpu:aarch64",
+    ],
+    define_values = {"pagesize": "64k"},
+)
@@ -71,9 +71,11 @@ def go_binary(name, static = False, pure = False, x_defs = None, **kwargs):
        "//conditions:default": kwargs.pop("gc_goopts", []),
        "//tools:debug": kwargs.pop("gc_goopts", []) + ["-N", "-l"],
    })
+    base_gotags = kwargs.pop("gotags", [])
    kwargs["gotags"] = select({
-        "//conditions:default": kwargs.pop("gotags", []),
-        "//tools:debug": kwargs.pop("gotags", []) + ["debug"],
+        "//tools/bazeldefs:pagesize_64k": base_gotags + ["pagesize_64k"],
+        "//tools:debug": base_gotags + ["debug"],
+        "//conditions:default": base_gotags,
    })
    _go_binary(
        name = name,
@@ -123,6 +125,12 @@ def go_test(name, static = False, pure = False, library = None, **kwargs):
        kwargs["static"] = "on"
    if library:
        kwargs["embed"] = [library]
+    base_gotags = kwargs.pop("gotags", [])
+    kwargs["gotags"] = select({
+        "//tools/bazeldefs:pagesize_64k": base_gotags + ["pagesize_64k"],
+        "//tools:debug": base_gotags + ["debug"],
+        "//conditions:default": base_gotags,
+    })
    _go_test(
        name = name,
        **kwargs
@@ -23,6 +23,7 @@ archs = [
    "_amd64",
    "_arm",
    "_arm64",
+    "_arm64_64k",
    "_mips",
    "_mips64",
    "_mips64le",