qemu: enable iommu on q35

Add a configuration option and a Pod Annotation

If activated:
- Add kernel parameters to load iommu
- Add irqchip=split in the kvm options
- Add a vIOMMU to the VM

Fixes #2694
Signed-off-by: Adrian Moreno <[email protected]>

cli/config/configuration-fc.toml.in

@@ -130,6 +130,12 @@ block_device_driver = "@DEFBLOCKSTORAGEDRIVER_FC@"
 # result in memory pre allocation
 #enable_hugepages = true
 
+# Enable vIOMMU, default false
+# Enabling this will result in the VM having a vIOMMU device
+# This will also add the following options to the kernel's
+# command line: intel_iommu=on,iommu=pt
+#enable_iommu = true
+
 # Enable swap of vm memory. Default false.
 # The behaviour is undefined if mem_prealloc is also set to true
 #enable_swap = true

cli/config/configuration-qemu-virtiofs.toml.in

@@ -189,6 +189,12 @@ enable_vhost_user_store = @DEFENABLEVHOSTUSERSTORE@
 # simulated block device nodes for vhost-user devices to live.
 vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@"
 
+# Enable vIOMMU, default false
+# Enabling this will result in the VM having a vIOMMU device
+# This will also add the following options to the kernel's
+# command line: intel_iommu=on,iommu=pt
+#enable_iommu = true
+
 # Enable file based guest memory support. The default is an empty string which
 # will disable this feature. In the case of virtio-fs, this is enabled
 # automatically and '/dev/shm' is used as the backing folder.

cli/config/configuration-qemu.toml.in

@@ -196,6 +196,12 @@ enable_vhost_user_store = @DEFENABLEVHOSTUSERSTORE@
 # simulated block device nodes for vhost-user devices to live.
 vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@"
 
+# Enable vIOMMU, default false
+# Enabling this will result in the VM having a vIOMMU device
+# This will also add the following options to the kernel's
+# command line: intel_iommu=on,iommu=pt
+#enable_iommu = true
+
 # Enable file based guest memory support. The default is an empty string which
 # will disable this feature. In the case of virtio-fs, this is enabled
 # automatically and '/dev/shm' is used as the backing folder.

pkg/katautils/config-settings.go.in

@@ -39,6 +39,7 @@ const defaultBlockDeviceCacheNoflush bool = false
 const defaultEnableIOThreads bool = false
 const defaultEnableMemPrealloc bool = false
 const defaultEnableHugePages bool = false
+const defaultEnableIOMMU bool = false
 const defaultFileBackedMemRootDir string = ""
 const defaultEnableSwap bool = false
 const defaultEnableDebug bool = false

pkg/katautils/config.go

@@ -120,6 +120,7 @@ type hypervisor struct {
 	MemPrealloc             bool     `toml:"enable_mem_prealloc"`
 	HugePages               bool     `toml:"enable_hugepages"`
 	VirtioMem               bool     `toml:"enable_virtio_mem"`
+	IOMMU                   bool     `toml:"enable_iommu"`
 	FileBackedMemRootDir    string   `toml:"file_mem_backend"`
 	Swap                    bool     `toml:"enable_swap"`
 	Debug                   bool     `toml:"enable_debug"`
@@ -668,6 +669,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		VirtioFSExtraArgs:       h.VirtioFSExtraArgs,
 		MemPrealloc:             h.MemPrealloc,
 		HugePages:               h.HugePages,
+		IOMMU:                   h.IOMMU,
 		FileBackedMemRootDir:    h.FileBackedMemRootDir,
 		Mlock:                   !h.Swap,
 		Debug:                   h.Debug,
@@ -1110,6 +1112,7 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig {
 		DefaultBridges:          defaultBridgesCount,
 		MemPrealloc:             defaultEnableMemPrealloc,
 		HugePages:               defaultEnableHugePages,
+		IOMMU:                   defaultEnableIOMMU,
 		FileBackedMemRootDir:    defaultFileBackedMemRootDir,
 		Mlock:                   !defaultEnableSwap,
 		Debug:                   defaultEnableDebug,

virtcontainers/hypervisor.go

@@ -361,6 +361,9 @@ type HypervisorConfig struct {
 	// VirtioMem is used to enable/disable virtio-mem
 	VirtioMem bool
 
+	// IOMMU specifies if the VM should have a vIOMMU
+	IOMMU bool
+
 	// Realtime Used to enable/disable realtime
 	Realtime bool
 

virtcontainers/pkg/annotations/annotations.go

@@ -151,6 +151,9 @@ const (
 	// HugePages is a sandbox annotation to specify if the memory should be pre-allocated from huge pages
 	HugePages = kataAnnotHypervisorPrefix + "enable_hugepages"
 
+	// Iommu is a sandbox annotation to specify if the VM should have a vIOMMU device
+	IOMMU = kataAnnotHypervisorPrefix + "enable_iommu"
+
 	// FileBackedMemRootDir is a sandbox annotation to soecify file based memory backend root directory
 	FileBackedMemRootDir = kataAnnotHypervisorPrefix + "file_mem_backend"
 

virtcontainers/pkg/oci/utils.go

@@ -545,6 +545,15 @@ func addHypervisorMemoryOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig
 
 		sbConfig.HypervisorConfig.HugePages = hugePages
 	}
+
+	if value, ok := ocispec.Annotations[vcAnnotations.IOMMU]; ok {
+		iommu, err := strconv.ParseBool(value)
+		if err != nil {
+			return fmt.Errorf("Error parsing annotation for iommu: Please specify boolean value 'true|false'")
+		}
+
+		sbConfig.HypervisorConfig.IOMMU = iommu
+	}
 	return nil
 }
 

virtcontainers/pkg/oci/utils_test.go

@@ -771,6 +771,7 @@ func TestAddHypervisorAnnotations(t *testing.T) {
 	ocispec.Annotations[vcAnnotations.EnableSwap] = "true"
 	ocispec.Annotations[vcAnnotations.FileBackedMemRootDir] = "/dev/shm"
 	ocispec.Annotations[vcAnnotations.HugePages] = "true"
+	ocispec.Annotations[vcAnnotations.IOMMU] = "true"
 	ocispec.Annotations[vcAnnotations.BlockDeviceDriver] = "virtio-scsi"
 	ocispec.Annotations[vcAnnotations.DisableBlockDeviceUse] = "true"
 	ocispec.Annotations[vcAnnotations.EnableIOThreads] = "true"
@@ -803,6 +804,7 @@ func TestAddHypervisorAnnotations(t *testing.T) {
 	assert.Equal(config.HypervisorConfig.Mlock, false)
 	assert.Equal(config.HypervisorConfig.FileBackedMemRootDir, "/dev/shm")
 	assert.Equal(config.HypervisorConfig.HugePages, true)
+	assert.Equal(config.HypervisorConfig.IOMMU, true)
 	assert.Equal(config.HypervisorConfig.BlockDeviceDriver, "virtio-scsi")
 	assert.Equal(config.HypervisorConfig.DisableBlockDeviceUse, true)
 	assert.Equal(config.HypervisorConfig.EnableIOThreads, true)

virtcontainers/qemu.go

@@ -409,6 +409,13 @@ func (q *qemu) buildDevices(initrdPath string) ([]govmmQemu.Device, *govmmQemu.I
 		}
 	}
 
+	if q.config.IOMMU {
+		devices, err = q.arch.appendIOMMU(devices)
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+
 	var ioThread *govmmQemu.IOThread
 	if q.config.BlockDeviceDriver == config.VirtioSCSI {
 		return q.arch.appendSCSIController(devices, q.config.EnableIOThreads)

virtcontainers/qemu_amd64.go

@@ -49,7 +49,6 @@ var kernelParams = []Param{
 	{"reboot", "k"},
 	{"console", "hvc0"},
 	{"console", "hvc1"},
-	{"iommu", "off"},
 	{"cryptomgr.notests", ""},
 	{"net.ifnames", "0"},
 	{"pci", "lastbus=0"},
@@ -94,12 +93,31 @@ func newQemuArch(config HypervisorConfig) qemuArch {
 		factory = true
 	}
 
+	var qemuMachines = supportedQemuMachines
+	if config.IOMMU {
+		var q35QemuIOMMUOptions = "accel=kvm,kernel_irqchip=split"
+
+		kernelParams = append(kernelParams,
+			Param{"intel_iommu", "on"})
+		kernelParams = append(kernelParams,
+			Param{"iommu", "pt"})
+
+		for _, m := range qemuMachines {
+			if m.Type == QemuQ35 {
+				m.Options = q35QemuIOMMUOptions
+			}
+		}
+	} else {
+		kernelParams = append(kernelParams,
+			Param{"iommu", "off"})
+	}
+
 	q := &qemuAmd64{
 		qemuArchBase: qemuArchBase{
 			machineType:           machineType,
 			memoryOffset:          config.MemOffset,
 			qemuPaths:             qemuPaths,
-			supportedQemuMachines: supportedQemuMachines,
+			supportedQemuMachines: qemuMachines,
 			kernelParamsNonDebug:  kernelParamsNonDebug,
 			kernelParamsDebug:     kernelParamsDebug,
 			kernelParams:          kernelParams,