From 54b24dad4db909d8a94b986a052131ae51500002 Mon Sep 17 00:00:00 2001 From: Liu Xiaodong Date: Sun, 12 Jan 2020 23:21:54 -0500 Subject: [PATCH] devices: add vhost-user storage configuration Two parameters are used to set in toml file: 1. Set "enable_vhost_user_store = true" to indicate that vhost-user storage device feature is enabled. 2. Set "vhost_user_store_path = ". vhost-user socket files will be under "/block/sockets/"; and device node for vhost-user device will be under "/block/devices/" The default value of "vhost_user_store_path" is "/var/run/kata-containers/vhost-user/". Fixes: #2380 Signed-off-by: Liu Xiaodong --- Makefile | 6 +++ .../configuration-qemu-virtiofs.toml.in | 11 +++++ cli/config/configuration-qemu.toml.in | 11 +++++ pkg/katautils/config-settings.go | 1 + pkg/katautils/config.go | 12 ++++++ pkg/katautils/config_test.go | 17 ++++++++ virtcontainers/hypervisor.go | 7 ++++ virtcontainers/persist.go | 4 ++ virtcontainers/persist/api/config.go | 7 ++++ virtcontainers/pkg/annotations/annotations.go | 7 ++++ virtcontainers/qemu.go | 40 +++++++++++++++---- virtcontainers/qemu_test.go | 27 +++++++++++++ 12 files changed, 143 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index ed2ede316a..5bb16f7cff 100644 --- a/Makefile +++ b/Makefile @@ -181,6 +181,8 @@ DEFVIRTIOFSEXTRAARGS := [] DEFENABLEIOTHREADS := false DEFENABLEMEMPREALLOC := false DEFENABLEHUGEPAGES := false +DEFENABLEVHOSTUSERSTORE := false +DEFVHOSTUSERSTOREPATH := $(PKGRUNDIR)/vhost-user DEFENABLESWAP := false DEFENABLEDEBUG := false DEFDISABLENESTINGCHECKS := false @@ -440,6 +442,8 @@ USER_VARS += DEFVIRTIOFSEXTRAARGS USER_VARS += DEFENABLEIOTHREADS USER_VARS += DEFENABLEMEMPREALLOC USER_VARS += DEFENABLEHUGEPAGES +USER_VARS += DEFENABLEVHOSTUSERSTORE +USER_VARS += DEFVHOSTUSERSTOREPATH USER_VARS += DEFENABLESWAP USER_VARS += DEFENABLEDEBUG USER_VARS += DEFDISABLENESTINGCHECKS @@ -622,6 +626,8 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit -e "s|@DEFENABLEIOTHREADS@|$(DEFENABLEIOTHREADS)|g" \ -e "s|@DEFENABLEMEMPREALLOC@|$(DEFENABLEMEMPREALLOC)|g" \ -e "s|@DEFENABLEHUGEPAGES@|$(DEFENABLEHUGEPAGES)|g" \ + -e "s|@DEFENABLEVHOSTUSERSTORE@|$(DEFENABLEVHOSTUSERSTORE)|g" \ + -e "s|@DEFVHOSTUSERSTOREPATH@|$(DEFVHOSTUSERSTOREPATH)|g" \ -e "s|@DEFENABLEMSWAP@|$(DEFENABLESWAP)|g" \ -e "s|@DEFENABLEDEBUG@|$(DEFENABLEDEBUG)|g" \ -e "s|@DEFDISABLENESTINGCHECKS@|$(DEFDISABLENESTINGCHECKS)|g" \ diff --git a/cli/config/configuration-qemu-virtiofs.toml.in b/cli/config/configuration-qemu-virtiofs.toml.in index a38e0a0d19..46a5ff0cab 100644 --- a/cli/config/configuration-qemu-virtiofs.toml.in +++ b/cli/config/configuration-qemu-virtiofs.toml.in @@ -172,6 +172,17 @@ enable_iothreads = @DEFENABLEIOTHREADS@ # result in memory pre allocation #enable_hugepages = true +# Enable vhost-user storage device, default false +# Enabling this will result in some Linux reserved block type +# major range 240-254 being chosen to represent vhost-user devices. +enable_vhost_user_store = @DEFENABLEVHOSTUSERSTORE@ + +# The base directory specifically used for vhost-user devices. +# Its sub-path "block" is used for block devices; "block/sockets" is +# where we expect vhost-user sockets to live; "block/devices" is where +# simulated block device nodes for vhost-user devices to live. +vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@" + # Enable file based guest memory support. The default is an empty string which # will disable this feature. In the case of virtio-fs, this is enabled # automatically and '/dev/shm' is used as the backing folder. diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in index 20e387008c..46ce7d9b53 100644 --- a/cli/config/configuration-qemu.toml.in +++ b/cli/config/configuration-qemu.toml.in @@ -179,6 +179,17 @@ enable_iothreads = @DEFENABLEIOTHREADS@ # result in memory pre allocation #enable_hugepages = true +# Enable vhost-user storage device, default false +# Enabling this will result in some Linux reserved block type +# major range 240-254 being chosen to represent vhost-user devices. +enable_vhost_user_store = @DEFENABLEVHOSTUSERSTORE@ + +# The base directory specifically used for vhost-user devices. +# Its sub-path "block" is used for block devices; "block/sockets" is +# where we expect vhost-user sockets to live; "block/devices" is where +# simulated block device nodes for vhost-user devices to live. +vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@" + # Enable file based guest memory support. The default is an empty string which # will disable this feature. In the case of virtio-fs, this is enabled # automatically and '/dev/shm' is used as the backing folder. diff --git a/pkg/katautils/config-settings.go b/pkg/katautils/config-settings.go index c46f36c15f..37464b93da 100644 --- a/pkg/katautils/config-settings.go +++ b/pkg/katautils/config-settings.go @@ -49,6 +49,7 @@ const defaultEntropySource = "/dev/urandom" const defaultGuestHookPath string = "" const defaultVirtioFSCacheMode = "none" const defaultDisableImageNvdimm = false +const defaultVhostUserStorePath string = "/var/run/kata-containers/vhost-user/" const defaultTemplatePath string = "/run/vc/vm/template" const defaultVMCacheEndpoint string = "/var/run/kata-containers/cache.sock" diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go index bf20a33206..b7b3ccd40d 100644 --- a/pkg/katautils/config.go +++ b/pkg/katautils/config.go @@ -104,6 +104,8 @@ type hypervisor struct { BlockDeviceCacheSet bool `toml:"block_device_cache_set"` BlockDeviceCacheDirect bool `toml:"block_device_cache_direct"` BlockDeviceCacheNoflush bool `toml:"block_device_cache_noflush"` + EnableVhostUserStore bool `toml:"enable_vhost_user_store"` + VhostUserStorePath string `toml:"vhost_user_store_path"` NumVCPUs int32 `toml:"default_vcpus"` DefaultMaxVCPUs uint32 `toml:"default_maxvcpus"` MemorySize uint32 `toml:"default_memory"` @@ -404,6 +406,13 @@ func (h hypervisor) guestHookPath() string { return h.GuestHookPath } +func (h hypervisor) vhostUserStorePath() string { + if h.VhostUserStorePath == "" { + return defaultVhostUserStorePath + } + return h.VhostUserStorePath +} + func (h hypervisor) getInitrdAndImage() (initrd string, image string, err error) { initrd, errInitrd := h.initrd() @@ -651,6 +660,8 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { HotplugVFIOOnRootBus: h.HotplugVFIOOnRootBus, PCIeRootPort: h.PCIeRootPort, DisableVhostNet: h.DisableVhostNet, + EnableVhostUserStore: h.EnableVhostUserStore, + VhostUserStorePath: h.vhostUserStorePath(), GuestHookPath: h.guestHookPath(), }, nil } @@ -1078,6 +1089,7 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig { HotplugVFIOOnRootBus: defaultHotplugVFIOOnRootBus, PCIeRootPort: defaultPCIeRootPort, GuestHookPath: defaultGuestHookPath, + VhostUserStorePath: defaultVhostUserStorePath, VirtioFSCache: defaultVirtioFSCacheMode, DisableImageNvdimm: defaultDisableImageNvdimm, } diff --git a/pkg/katautils/config_test.go b/pkg/katautils/config_test.go index eb15e5f3e7..221a4b5551 100644 --- a/pkg/katautils/config_test.go +++ b/pkg/katautils/config_test.go @@ -165,6 +165,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf MemSlots: defaultMemSlots, EntropySource: defaultEntropySource, GuestHookPath: defaultGuestHookPath, + VhostUserStorePath: defaultVhostUserStorePath, SharedFS: sharedFS, VirtioFSDaemon: "/path/to/virtiofsd", VirtioFSCache: defaultVirtioFSCacheMode, @@ -628,6 +629,7 @@ func TestMinimalRuntimeConfig(t *testing.T) { BlockDeviceDriver: defaultBlockDeviceDriver, Msize9p: defaultMsize9p, GuestHookPath: defaultGuestHookPath, + VhostUserStorePath: defaultVhostUserStorePath, VirtioFSCache: defaultVirtioFSCacheMode, } @@ -1198,6 +1200,21 @@ func TestHypervisorDefaultsGuestHookPath(t *testing.T) { assert.Equal(guestHookPath, testGuestHookPath, "custom guest hook path wrong") } +func TestHypervisorDefaultsVhostUserStorePath(t *testing.T) { + assert := assert.New(t) + + h := hypervisor{} + vhostUserStorePath := h.vhostUserStorePath() + assert.Equal(vhostUserStorePath, defaultVhostUserStorePath, "default vhost-user store path wrong") + + testVhostUserStorePath := "/test/vhost/user/store/path" + h = hypervisor{ + VhostUserStorePath: testVhostUserStorePath, + } + vhostUserStorePath = h.vhostUserStorePath() + assert.Equal(vhostUserStorePath, testVhostUserStorePath, "custom vhost-user store path wrong") +} + func TestProxyDefaults(t *testing.T) { assert := assert.New(t) diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go index 8b6a6cb8ac..ffa8f0fbbf 100644 --- a/virtcontainers/hypervisor.go +++ b/virtcontainers/hypervisor.go @@ -393,6 +393,13 @@ type HypervisorConfig struct { // DisableVhostNet is used to indicate if host supports vhost_net DisableVhostNet bool + // EnableVhostUserStore is used to indicate if host supports vhost-user-blk/scsi + EnableVhostUserStore bool + + // VhostUserStorePath is the directory path where vhost-user devices + // related folders, sockets and device nodes should be. + VhostUserStorePath string + // GuestHookPath is the path within the VM that will be used for 'drop-in' hooks GuestHookPath string diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go index 2793fe1d52..d96a3890ea 100644 --- a/virtcontainers/persist.go +++ b/virtcontainers/persist.go @@ -252,6 +252,8 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) { BootToBeTemplate: sconfig.HypervisorConfig.BootToBeTemplate, BootFromTemplate: sconfig.HypervisorConfig.BootFromTemplate, DisableVhostNet: sconfig.HypervisorConfig.DisableVhostNet, + EnableVhostUserStore: sconfig.HypervisorConfig.EnableVhostUserStore, + VhostUserStorePath: sconfig.HypervisorConfig.VhostUserStorePath, GuestHookPath: sconfig.HypervisorConfig.GuestHookPath, VMid: sconfig.HypervisorConfig.VMid, } @@ -539,6 +541,8 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) { BootToBeTemplate: hconf.BootToBeTemplate, BootFromTemplate: hconf.BootFromTemplate, DisableVhostNet: hconf.DisableVhostNet, + EnableVhostUserStore: hconf.EnableVhostUserStore, + VhostUserStorePath: hconf.VhostUserStorePath, GuestHookPath: hconf.GuestHookPath, VMid: hconf.VMid, } diff --git a/virtcontainers/persist/api/config.go b/virtcontainers/persist/api/config.go index 7912587057..34a5fd0fbf 100644 --- a/virtcontainers/persist/api/config.go +++ b/virtcontainers/persist/api/config.go @@ -166,6 +166,13 @@ type HypervisorConfig struct { // DisableVhostNet is used to indicate if host supports vhost_net DisableVhostNet bool + // EnableVhostUserStore is used to indicate if host supports vhost-user-blk/scsi + EnableVhostUserStore bool + + // VhostUserStorePath is the directory path where vhost-user devices + // related folders, sockets and device nodes should be. + VhostUserStorePath string + // GuestHookPath is the path within the VM that will be used for 'drop-in' hooks GuestHookPath string diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go index f18e3ffed5..10ce7833e2 100644 --- a/virtcontainers/pkg/annotations/annotations.go +++ b/virtcontainers/pkg/annotations/annotations.go @@ -84,6 +84,13 @@ const ( // DisableVhostNet is a sandbox annotation to specify if vhost-net is not available on the host. DisableVhostNet = kataAnnotHypervisorPrefix + "disable_vhost_net" + // EnableVhostUserStore is a sandbox annotation to specify if vhost-user-blk/scsi is abailable on the host + EnableVhostUserStore = kataAnnotHypervisorPrefix + "enable_vhost_user_store" + + // VhostUserStorePath is a sandbox annotation to specify the directory path where vhost-user devices + // related folders, sockets and device nodes should be. + VhostUserStorePath = kataAnnotHypervisorPrefix + "vhost_user_store_path" + // GuestHookPath is a sandbox annotation to specify the path within the VM that will be used for 'drop-in' hooks. GuestHookPath = kataAnnotHypervisorPrefix + "guest_hook_path" diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index 002b208294..3fde966696 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -522,6 +522,15 @@ func (q *qemu) createSandbox(ctx context.Context, id string, networkNS NetworkNa } } + // Vhost-user-blk/scsi process which can improve performance, like SPDK, + // requires shared-on hugepage to work with Qemu. + if q.config.EnableVhostUserStore { + if !q.config.HugePages { + return errors.New("Vhost-user-blk/scsi is enabled without HugePages. This configuration will not work") + } + knobs.MemShared = true + } + rtc := govmmQemu.RTC{ Base: "utc", DriftFix: "slew", @@ -678,7 +687,7 @@ func (q *qemu) setupVirtiofsd() (err error) { return err } -func (q *qemu) getMemArgs() (bool, string, string) { +func (q *qemu) getMemArgs() (bool, string, string, error) { share := false target := "" memoryBack := "memory-backend-ram" @@ -689,15 +698,24 @@ func (q *qemu) getMemArgs() (bool, string, string) { target = "/dev/hugepages" memoryBack = "memory-backend-file" share = true - } else if q.config.SharedFS == config.VirtioFS || q.config.FileBackedMemRootDir != "" { - target = q.qemuConfig.Memory.Path - memoryBack = "memory-backend-file" + } else { + if q.config.EnableVhostUserStore { + // Vhost-user-blk/scsi process which can improve performance, like SPDK, + // requires shared-on hugepage to work with Qemu. + return share, target, "", fmt.Errorf("Vhost-user-blk/scsi requires hugepage memory") + } + + if q.config.SharedFS == config.VirtioFS || q.config.FileBackedMemRootDir != "" { + target = q.qemuConfig.Memory.Path + memoryBack = "memory-backend-file" + } } + if q.qemuConfig.Knobs.MemShared { share = true } - return share, target, memoryBack + return share, target, memoryBack, nil } func (q *qemu) setupVirtioMem() error { @@ -708,7 +726,11 @@ func (q *qemu) setupVirtioMem() error { // 1024 is size for nvdimm sizeMB := int(maxMem) - int(q.config.MemorySize) - share, target, memoryBack := q.getMemArgs() + share, target, memoryBack, err := q.getMemArgs() + if err != nil { + return err + } + err = q.qmpSetup() if err != nil { return err @@ -1551,7 +1573,11 @@ func (q *qemu) hotplugAddMemory(memDev *memoryDevice) (int, error) { memDev.slot = maxSlot + 1 } - share, target, memoryBack := q.getMemArgs() + share, target, memoryBack, err := q.getMemArgs() + if err != nil { + return 0, err + } + err = q.qmpMonitorCh.qmp.ExecHotplugMemory(q.qmpMonitorCh.ctx, memoryBack, "mem"+strconv.Itoa(memDev.slot), target, memDev.sizeMB, share) if err != nil { q.Logger().WithError(err).Error("hotplug memory") diff --git a/virtcontainers/qemu_test.go b/virtcontainers/qemu_test.go index 5c6eec7576..b7dafb3013 100644 --- a/virtcontainers/qemu_test.go +++ b/virtcontainers/qemu_test.go @@ -465,6 +465,33 @@ func TestQemuFileBackedMem(t *testing.T) { assert.Equal(q.qemuConfig.Knobs.FileBackedMem, false) assert.Equal(q.qemuConfig.Knobs.MemShared, false) assert.Equal(q.qemuConfig.Memory.Path, "") + + // Check setting vhost-user storage with Hugepages + sandbox, err = createQemuSandboxConfig() + assert.NoError(err) + + q = &qemu{ + store: sandbox.newStore, + } + sandbox.config.HypervisorConfig.EnableVhostUserStore = true + sandbox.config.HypervisorConfig.HugePages = true + err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, false) + assert.NoError(err) + assert.Equal(q.qemuConfig.Knobs.MemShared, true) + + // Check failure for vhost-user storage + sandbox, err = createQemuSandboxConfig() + assert.NoError(err) + + q = &qemu{ + store: sandbox.newStore, + } + sandbox.config.HypervisorConfig.EnableVhostUserStore = true + sandbox.config.HypervisorConfig.HugePages = false + err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, false) + + expectErr = errors.New("Vhost-user-blk/scsi is enabled without HugePages. This configuration will not work") + assert.Equal(expectErr.Error(), err.Error()) } func createQemuSandboxConfig() (*Sandbox, error) {