Files
visor/pkg/sentry/socket/plugin/stack/readwriter.go
T
Anqi Shen 3b519026ca Support plugin network stack
This commit supports a third-party network stack as a plugin stack for
gVisor.

The overall plugin package structure is the following:

- pkg/sentry/socket/plugin:
  Interfaces for initializing plugin network stack. It will be used
  in network setting up during sandbox creating.

- pkg/sentry/socket/plugin/stack:
  Glue layer for plugin stack's socket and stack ops with sentry. It
  will also register plugin stack operations if imported.

- pkg/sentry/socket/plugin/cgo:
  Interfaces defined in C for plugin network stack to support.

To build target runsc-plugin-stack, which imports
pkg/sentry/socket/plugin/stack package and enables CGO:

bazel build --config=plugin-tldk runsc:runsc-plugin-stack

(i.e. --config=plugin-tldk indicates that using TLDK as plugin stack)

By using runsc-plugin-stack binary and setting "--network=plugin" in
runtimeArgs, user can use third-party network stack instead of
netstack embedded in gVisor to get better network performance.

Redis benchmark with following setups:
1. KVM platform
2. 4 physical cores for target pod
3. target pod as redis server

Runc:
$redis-benchmark -h [target ip] -n 100000 -t get,set -q
SET: 115207.38 requests per second, p50=0.215 msec
GET: 92336.11 requests per second, p50=0.279 msec

$redis-benchmark -h [target ip] -n 100000 -t get,set -q
SET: 113895.21 requests per second, p50=0.247 msec
GET: 96899.23 requests per second, p50=0.271 msec

$redis-benchmark -h [target ip] -n 100000 -t get,set -q
SET: 126582.27 requests per second, p50=0.199 msec
GET: 95969.28 requests per second, p50=0.271 msec

Runsc with plugin stack:
$redis-benchmark -h [target ip] -n 100000 -t get,set -q
SET: 123915.74 requests per second, p50=0.343 msec
GET: 115473.45 requests per second, p50=0.335 msec

$redis-benchmark -h [target ip] -n 100000 -t get,set -q
SET: 120918.98 requests per second, p50=0.351 msec
GET: 117647.05 requests per second, p50=0.351 msec

$redis-benchmark -h [target ip] -n 100000 -t get,set -q
SET: 119904.08 requests per second, p50=0.367 msec
GET: 112739.57 requests per second, p50=0.375 msec

Runsc with netstack:
$redis-benchmark -h [target ip] -n 100000 -t get,set -q
SET: 59952.04 requests per second, p50=0.759 msec
GET: 61162.08 requests per second, p50=0.631 msec

$redis-benchmark -h [target ip] -n 100000 -t get,set -q
SET: 52219.32 requests per second, p50=0.719 msec
GET: 58719.91 requests per second, p50=0.663 msec

$redis-benchmark -h [target ip] -n 100000 -t get,set -q
SET: 59952.04 requests per second, p50=0.751 msec
GET: 60827.25 requests per second, p50=0.751 msec

Updates https://github.com/google/gvisor/issues/9266

Co-developed-by: Tianyu Zhou <[email protected]>
Signed-off-by: Anqi Shen <[email protected]>
2024-07-12 09:10:13 +00:00

111 lines
3.0 KiB
Go

// Copyright 2023 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package stack
import (
"sync"
"syscall"
"gvisor.dev/gvisor/pkg/abi/linux"
"gvisor.dev/gvisor/pkg/safemem"
"gvisor.dev/gvisor/pkg/sentry/socket/plugin/cgo"
)
type pluginStackRW struct {
handle uint32
// Represent both input and output flags.
flags uint32
// Reused as msg_control for read.
to []byte
iovs [3]syscall.Iovec
}
var pluginStackRWPool = sync.Pool{
New: func() interface{} {
return &pluginStackRW{}
},
}
func getReadWriter(handle uint32) *pluginStackRW {
rw := pluginStackRWPool.Get().(*pluginStackRW)
rw.handle = handle
return rw
}
func putReadWriter(rw *pluginStackRW) {
*rw = pluginStackRW{}
pluginStackRWPool.Put(rw)
}
// ReadToBlocks implements safemem.Reader.ReadToBlocks.
func (rw *pluginStackRW) ReadToBlocks(dsts safemem.BlockSeq) (uint64, error) {
// Set MSG_DONTWAIT flag to avoid blocking in plugin stack.
flags := int(rw.flags) & ^linux.MSG_DONTWAIT
if len(rw.to) != 0 || flags != 0 {
iovs := iovecsFromBlockSeq(dsts, rw)
rc, _, lc, mflags := cgo.Recvmsg(rw.handle, iovs, nil, rw.to, int(rw.flags))
if rc >= 0 {
rw.to = rw.to[:lc]
rw.flags = uint32(mflags)
}
return translateReturn(rc)
}
var rc int64
if dsts.IsEmpty() {
rc = 0
} else if dsts.NumBlocks() == 1 {
rc = cgo.Read(rw.handle, dsts.Head().Addr(), dsts.Head().Len())
} else {
rc = cgo.Readv(rw.handle, iovecsFromBlockSeq(dsts, rw))
}
return translateReturn(rc)
}
// WriteFromBlocks implements safemem.Writer.WriteFromBlocks.
//
// Preconditions: rw.d.metadataMu must be locked.
func (rw *pluginStackRW) WriteFromBlocks(srcs safemem.BlockSeq) (uint64, error) {
var rc int64
if rw.to != nil {
if srcs.IsEmpty() {
// Invoke plugin stack checking whether there is any error to report
// on target socket which sends 0-length data.
rc = cgo.Sendto(rw.handle, 0, 0, 0, rw.to)
} else if srcs.NumBlocks() == 1 {
rc = cgo.Sendto(rw.handle, srcs.Head().Addr(), srcs.Head().Len(), 0, rw.to)
} else {
iovs := iovecsFromBlockSeq(srcs, rw)
rc = cgo.Sendmsg(rw.handle, iovs, rw.to, 0)
}
} else {
if srcs.IsEmpty() {
// Invoke plugin stack checking whether there is any error to report
// on target socket which sends 0-length data.
rc = cgo.Write(rw.handle, 0, 0)
} else if srcs.NumBlocks() == 1 {
rc = cgo.Write(rw.handle, srcs.Head().Addr(), srcs.Head().Len())
} else {
rc = cgo.Writev(rw.handle, iovecsFromBlockSeq(srcs, rw))
}
}
return translateReturn(rc)
}