mirror of
https://github.com/hanzoai/visor.git
synced 2026-09-24 17:04:14 +00:00
Ayush RanjanandAlessio Ricci Toniolo
1f926a2ab7
Containers with /dev/infiniband/uverbs* devices need a sysfs surface for device discovery: libibverbs enumerates /sys/class/infiniband_verbs, and NCCL builds its hardware topology by resolving /sys/class symlinks with realpath() and walking the PCI hierarchy. This adds a --rdmaproxy flag which will gate all RDMA support. For now it only wires up the sysfs topology: runsc snapshots the host sysfs closure derived from the spec's uverbs devices (ibdevs, netdevs, PCI ancestor chains, GPU functions, NUMA nodes) before pivot_root, and the sentry constructs a path-identical virtual tree: one canonical /sys/devices/pci... subtree per device, with /sys/class/* and /sys/bus/pci/devices as symlink farms matching the kernel layout. Per-port dynamic state (RoCE GID tables, link state, counters) is served live through targeted read-only bind mounts of the per-ibdev ports subtrees; identity and PCI attributes are a static snapshot, since the collecting process and the constructed sandbox tree do not observe the same sysfs mount (the RDMA netdevs move into the sandbox network namespace only after the sandbox is created). Each PCI function directory also gets a subsystem symlink into a synthesized /sys/bus/pci, and every node's bus gets a pci_bus entry: NCCL classifies a directory as a PCI device by following its subsystem symlink, and aborts topology discovery without it. The NUMA topology is not mirrored: the sentry advertises a single node holding every sandbox CPU, consistent with its single-node memory model (set_mempolicy's maxNodes == 1) and its single-socket /proc/cpuinfo view. Host node cpumasks name CPUs the sandbox does not have, which breaks libnuma/hwloc consumers whenever the sandbox CPU count is below the host's (numactl node binding, which NUMA-aware stacks exec in front of worker subprocesses, kills the child). The PCI numa_node, local_cpus and local_cpulist attributes are rewritten to match the single-node view. The verbs device proxy that makes RDMA usable end-to-end is not part of this change; --rdmaproxy is a work in progress and RDMA is not yet functional. Co-authored-by: Alessio Ricci Toniolo <[email protected]>
74 lines
1.6 KiB
Python
74 lines
1.6 KiB
Python
load("//tools:defs.bzl", "go_library", "go_test")
|
|
load("//tools/go_generics:defs.bzl", "go_template_instance")
|
|
|
|
package(default_applicable_licenses = ["//:license"])
|
|
|
|
licenses(["notice"])
|
|
|
|
go_template_instance(
|
|
name = "dir_refs",
|
|
out = "dir_refs.go",
|
|
package = "sys",
|
|
prefix = "dir",
|
|
template = "//pkg/refs:refs_template",
|
|
types = {
|
|
"T": "dir",
|
|
},
|
|
)
|
|
|
|
go_library(
|
|
name = "sys",
|
|
srcs = [
|
|
"dir_refs.go",
|
|
"kcov.go",
|
|
"pci.go",
|
|
"rdma.go",
|
|
"save_restore.go",
|
|
"sys.go",
|
|
],
|
|
visibility = ["//pkg/sentry:internal"],
|
|
deps = [
|
|
"//pkg/abi/linux",
|
|
"//pkg/atomicbitops",
|
|
"//pkg/context",
|
|
"//pkg/coverage",
|
|
"//pkg/errors/linuxerr",
|
|
"//pkg/fspath",
|
|
"//pkg/fsutil",
|
|
"//pkg/log",
|
|
"//pkg/rdma",
|
|
"//pkg/refs",
|
|
"//pkg/sentry/arch",
|
|
"//pkg/sentry/fsimpl/host",
|
|
"//pkg/sentry/fsimpl/kernfs",
|
|
"//pkg/sentry/inet",
|
|
"//pkg/sentry/kernel",
|
|
"//pkg/sentry/kernel/auth",
|
|
"//pkg/sentry/memmap",
|
|
"//pkg/sentry/vfs",
|
|
"//pkg/usermem",
|
|
"@org_golang_x_sys//unix:go_default_library",
|
|
],
|
|
)
|
|
|
|
go_test(
|
|
name = "sys_test",
|
|
srcs = ["sys_test.go"],
|
|
library = ":sys",
|
|
)
|
|
|
|
go_test(
|
|
name = "sys_integration_test",
|
|
srcs = ["sys_integration_test.go"],
|
|
deps = [
|
|
":sys",
|
|
"//pkg/abi/linux",
|
|
"//pkg/rdma",
|
|
"//pkg/sentry/fsimpl/testutil",
|
|
"//pkg/sentry/kernel",
|
|
"//pkg/sentry/kernel/auth",
|
|
"//pkg/sentry/vfs",
|
|
"@com_github_google_go_cmp//cmp:go_default_library",
|
|
],
|
|
)
|