runtime: Enable file based backend

A file based memory backend mapped to the host, fot eg: '/dev/shm' will
be used by virtio-fs for performance reasons. This change is a generic
implementation of that for kata. This will be enabled default for
virtio-fs negating the need to enable hugepages in that scenario. This
option can be used without virtio-fs by setting 'file_mem_backend' to
the location in the configuration file. Default value is an empty
string.

Fixes: #1656
Signed-off-by: Ganesh Maharaj Mahalingam <[email protected]>

cli/config/configuration-qemu.toml.in

@@ -165,6 +165,12 @@ enable_iothreads = @DEFENABLEIOTHREADS@
 # result in memory pre allocation
 #enable_hugepages = true
 
+# Enable file based guest memory support. The default is an empty string which
+# will disable this feature. In the case of virtio-fs, this is enabled
+# automatically and '/dev/shm' is used as the backing folder.
+# This option will be ignored if VM templating is enabled.
+#file_mem_backend = ""
+
 # Enable swap of vm memory. Default false.
 # The behaviour is undefined if mem_prealloc is also set to true
 #enable_swap = true

pkg/katautils/config-settings.go

@@ -35,6 +35,7 @@ const defaultBlockDeviceCacheNoflush bool = false
 const defaultEnableIOThreads bool = false
 const defaultEnableMemPrealloc bool = false
 const defaultEnableHugePages bool = false
+const defaultFileBackedMemRootDir string = ""
 const defaultEnableSwap bool = false
 const defaultEnableDebug bool = false
 const defaultDisableNestingChecks bool = false

pkg/katautils/config.go

@@ -109,6 +109,7 @@ type hypervisor struct {
 	DisableBlockDeviceUse   bool   `toml:"disable_block_device_use"`
 	MemPrealloc             bool   `toml:"enable_mem_prealloc"`
 	HugePages               bool   `toml:"enable_hugepages"`
+	FileBackedMemRootDir    string `toml:"file_mem_backend"`
 	Swap                    bool   `toml:"enable_swap"`
 	Debug                   bool   `toml:"enable_debug"`
 	DisableNestingChecks    bool   `toml:"disable_nesting_checks"`
@@ -584,6 +585,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		VirtioFSCache:           h.VirtioFSCache,
 		MemPrealloc:             h.MemPrealloc,
 		HugePages:               h.HugePages,
+		FileBackedMemRootDir:    h.FileBackedMemRootDir,
 		Mlock:                   !h.Swap,
 		Debug:                   h.Debug,
 		DisableNestingChecks:    h.DisableNestingChecks,
@@ -850,6 +852,7 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig {
 		DefaultBridges:          defaultBridgesCount,
 		MemPrealloc:             defaultEnableMemPrealloc,
 		HugePages:               defaultEnableHugePages,
+		FileBackedMemRootDir:    defaultFileBackedMemRootDir,
 		Mlock:                   !defaultEnableSwap,
 		Debug:                   defaultEnableDebug,
 		DisableNestingChecks:    defaultDisableNestingChecks,

virtcontainers/hypervisor.go

@@ -263,6 +263,9 @@ type HypervisorConfig struct {
 	// HugePages specifies if the memory should be pre-allocated from huge pages
 	HugePages bool
 
+	// File based memory backend root directory
+	FileBackedMemRootDir string
+
 	// Realtime Used to enable/disable realtime
 	Realtime bool
 

virtcontainers/persist/api/config.go

@@ -121,6 +121,9 @@ type HypervisorConfig struct {
 	// HugePages specifies if the memory should be pre-allocated from huge pages
 	HugePages bool
 
+	// File based memory backend root directory
+	FileBackedMemRootDir string
+
 	// Realtime Used to enable/disable realtime
 	Realtime bool
 

virtcontainers/qemu.go

@@ -96,9 +96,10 @@ const (
 	qmpCapErrMsg  = "Failed to negoatiate QMP capabilities"
 	qmpExecCatCmd = "exec:cat"
 
-	scsiControllerID  = "scsi0"
-	rngID             = "rng0"
-	vsockKernelOption = "agent.use_vsock"
+	scsiControllerID         = "scsi0"
+	rngID                    = "rng0"
+	vsockKernelOption        = "agent.use_vsock"
+	fallbackFileBackedMemDir = "/dev/shm"
 )
 
 var qemuMajorVersion int
@@ -423,6 +424,23 @@ func (q *qemu) setupTemplate(knobs *govmmQemu.Knobs, memory *govmmQemu.Memory) g
 	return incoming
 }
 
+func (q *qemu) setupFileBackedMem(knobs *govmmQemu.Knobs, memory *govmmQemu.Memory) {
+	var target string
+	if q.config.FileBackedMemRootDir != "" {
+		target = q.config.FileBackedMemRootDir
+	} else {
+		target = fallbackFileBackedMemDir
+	}
+	if _, err := os.Stat(target); err != nil {
+		q.Logger().WithError(err).Error("File backed memory location does not exist")
+		return
+	}
+
+	knobs.FileBackedMem = true
+	knobs.FileBackedMemShared = true
+	memory.Path = target
+}
+
 // createSandbox is the Hypervisor sandbox creation implementation for govmmQemu.
 func (q *qemu) createSandbox(ctx context.Context, id string, hypervisorConfig *HypervisorConfig, store *store.VCStore) error {
 	// Save the tracing context
@@ -476,6 +494,19 @@ func (q *qemu) createSandbox(ctx context.Context, id string, hypervisorConfig *H
 
 	incoming := q.setupTemplate(&knobs, &memory)
 
+	// With the current implementations, VM templating will not work with file
+	// based memory (stand-alone) or virtiofs. This is because VM templating
+	// builds the first VM with file-backed memory and shared=on and the
+	// subsequent ones with shared=off. virtio-fs always requires shared=on for
+	// memory.
+	if q.config.SharedFS == config.VirtioFS || q.config.FileBackedMemRootDir != "" {
+		if !(q.config.BootToBeTemplate || q.config.BootFromTemplate) {
+			q.setupFileBackedMem(&knobs, &memory)
+		} else {
+			return errors.New("VM templating has been enabled with either virtio-fs or file backed memory and this configuration will not work")
+		}
+	}
+
 	rtc := govmmQemu.RTC{
 		Base:     "utc",
 		DriftFix: "slew",

virtcontainers/qemu_test.go

@@ -16,6 +16,7 @@ import (
 	"testing"
 
 	govmmQemu "github.com/intel/govmm/qemu"
+	"github.com/kata-containers/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/runtime/virtcontainers/store"
 	"github.com/kata-containers/runtime/virtcontainers/types"
 	"github.com/stretchr/testify/assert"
@@ -482,3 +483,70 @@ func TestQemuAddDeviceToBridge(t *testing.T) {
 	exceptErr = errors.New("failed to get available address from bridges")
 	assert.Equal(exceptErr, err)
 }
+
+func TestQemuFileBackedMem(t *testing.T) {
+	assert := assert.New(t)
+
+	// Check default Filebackedmem location for virtio-fs
+	sandbox, err := createQemuSandboxConfig()
+	if err != nil {
+		t.Fatal(err)
+	}
+	q := &qemu{}
+	sandbox.config.HypervisorConfig.SharedFS = config.VirtioFS
+	if err = q.createSandbox(context.Background(), sandbox.id, &sandbox.config.HypervisorConfig, sandbox.store); err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(q.qemuConfig.Knobs.FileBackedMem, true)
+	assert.Equal(q.qemuConfig.Knobs.FileBackedMemShared, true)
+	assert.Equal(q.qemuConfig.Memory.Path, fallbackFileBackedMemDir)
+
+	// Check failure for VM templating
+	sandbox, err = createQemuSandboxConfig()
+	if err != nil {
+		t.Fatal(err)
+	}
+	q = &qemu{}
+	sandbox.config.HypervisorConfig.BootToBeTemplate = true
+	sandbox.config.HypervisorConfig.SharedFS = config.VirtioFS
+	sandbox.config.HypervisorConfig.MemoryPath = fallbackFileBackedMemDir
+
+	err = q.createSandbox(context.Background(), sandbox.id, &sandbox.config.HypervisorConfig, sandbox.store)
+
+	expectErr := errors.New("VM templating has been enabled with either virtio-fs or file backed memory and this configuration will not work")
+	assert.Equal(expectErr, err)
+
+	// Check Setting of non-existent shared-mem path
+	sandbox, err = createQemuSandboxConfig()
+	if err != nil {
+		t.Fatal(err)
+	}
+	q = &qemu{}
+	sandbox.config.HypervisorConfig.FileBackedMemRootDir = "/tmp/xyzabc"
+	if err = q.createSandbox(context.Background(), sandbox.id, &sandbox.config.HypervisorConfig, sandbox.store); err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(q.qemuConfig.Knobs.FileBackedMem, false)
+	assert.Equal(q.qemuConfig.Knobs.FileBackedMemShared, false)
+	assert.Equal(q.qemuConfig.Memory.Path, "")
+}
+
+func createQemuSandboxConfig() (*Sandbox, error) {
+
+	qemuConfig := newQemuConfig()
+	sandbox := Sandbox{
+		ctx: context.Background(),
+		id:  "testSandbox",
+		config: &SandboxConfig{
+			HypervisorConfig: qemuConfig,
+		},
+	}
+
+	vcStore, err := store.NewVCSandboxStore(sandbox.ctx, sandbox.id)
+	if err != nil {
+		return &Sandbox{}, err
+	}
+	sandbox.store = vcStore
+
+	return &sandbox, nil
+}