device: Add GPU device support

Enable GPU device support in kata runtime, including GVT-g and GVT-d.
GVT-g: graphic virtualization technology with mediated pass through
GVT-d: graphic virtualization technology with direct pass through

BDF of device eg "0000:00:1c.0" is used to distinguish GPU device in GVT-d,
while sysfsdev of device eg "f79944e4-5a3d-11e8-99ce-479cbab002e4" is used
in GVT-g.

Fixes kata-containers#542

Signed-off-by: Zhao Xinda <[email protected]>

virtcontainers/device/config/config.go

@@ -109,12 +109,33 @@ type BlockDrive struct {
 	VirtPath string
 }
 
+// VFIODeviceType indicates VFIO device type
+type VFIODeviceType uint32
+
+const (
+	// VFIODeviceErrorType is the error type of VFIO device
+	VFIODeviceErrorType VFIODeviceType = iota
+
+	// VFIODeviceNormalType is a normal VFIO device type
+	VFIODeviceNormalType
+
+	// VFIODeviceMediatedType is a VFIO mediated device type
+	VFIODeviceMediatedType
+)
+
 // VFIODev represents a VFIO drive used for hotplugging
 type VFIODev struct {
 	// ID is used to identify this drive in the hypervisor options.
 	ID string
+
+	// Type of VFIO device
+	Type VFIODeviceType
+
 	// BDF (Bus:Device.Function) of the PCI address
 	BDF string
+
+	// sysfsdev of VFIO mediated device
+	SysfsDev string
 }
 
 // RNGDev represents a random number generator device

virtcontainers/device/drivers/vfio.go

@@ -67,13 +67,15 @@ func (device *VFIODevice) Attach(devReceiver api.DeviceReceiver) error {
 	// Pass all devices in iommu group
 	for i, deviceFile := range deviceFiles {
 		//Get bdf of device eg 0000:00:1c.0
-		deviceBDF, err := getBDF(deviceFile.Name())
+		deviceBDF, deviceSysfsDev, vfioDeviceType, err := getVFIODetails(deviceFile.Name(), iommuDevicesPath)
 		if err != nil {
 			return err
 		}
 		vfio := &config.VFIODev{
-			ID:  utils.MakeNameID("vfio", device.DeviceInfo.ID+strconv.Itoa(i), maxDevIDSize),
-			BDF: deviceBDF,
+			ID:       utils.MakeNameID("vfio", device.DeviceInfo.ID+strconv.Itoa(i), maxDevIDSize),
+			Type:     vfioDeviceType,
+			BDF:      deviceBDF,
+			SysfsDev: deviceSysfsDev,
 		}
 		device.vfioDevs = append(device.vfioDevs, vfio)
 	}
@@ -130,17 +132,45 @@ func (device *VFIODevice) GetDeviceInfo() interface{} {
 // It should implement GetAttachCount() and DeviceID() as api.Device implementation
 // here it shares function from *GenericDevice so we don't need duplicate codes
 
-// getBDF returns the BDF of pci device
-// Expected input strng format is [<domain>]:[<bus>][<slot>].[<func>] eg. 0000:02:10.0
-func getBDF(deviceSysStr string) (string, error) {
-	tokens := strings.Split(deviceSysStr, ":")
+func getVFIODetails(deviceFileName, iommuDevicesPath string) (deviceBDF, deviceSysfsDev string, vfioDeviceType config.VFIODeviceType, err error) {
+	tokens := strings.Split(deviceFileName, ":")
+	vfioDeviceType = config.VFIODeviceErrorType
+	if len(tokens) == 3 {
+		vfioDeviceType = config.VFIODeviceNormalType
+	} else {
+		tokens = strings.Split(deviceFileName, "-")
+		if len(tokens) == 5 {
+			vfioDeviceType = config.VFIODeviceMediatedType
+		}
+	}
 
-	if len(tokens) != 3 {
-		return "", fmt.Errorf("Incorrect number of tokens found while parsing bdf for device : %s", deviceSysStr)
+	switch vfioDeviceType {
+	case config.VFIODeviceNormalType:
+		// Get bdf of device eg. 0000:00:1c.0
+		deviceBDF = getBDF(deviceFileName)
+	case config.VFIODeviceMediatedType:
+		// Get sysfsdev of device eg. /sys/devices/pci0000:00/0000:00:02.0/f79944e4-5a3d-11e8-99ce-479cbab002e4
+		sysfsDevStr := filepath.Join(iommuDevicesPath, deviceFileName)
+		deviceSysfsDev, err = getSysfsDev(sysfsDevStr)
+	default:
+		err = fmt.Errorf("Incorrect tokens found while parsing vfio details: %s", deviceFileName)
 	}
 
-	tokens = strings.SplitN(deviceSysStr, ":", 2)
-	return tokens[1], nil
+	return deviceBDF, deviceSysfsDev, vfioDeviceType, err
+}
+
+// getBDF returns the BDF of pci device
+// Expected input string format is [<domain>]:[<bus>][<slot>].[<func>] eg. 0000:02:10.0
+func getBDF(deviceSysStr string) string {
+	tokens := strings.SplitN(deviceSysStr, ":", 2)
+	return tokens[1]
+}
+
+// getSysfsDev returns the sysfsdev of mediated device
+// Expected input string format is absolute path to the sysfs dev node
+// eg. /sys/kernel/iommu_groups/0/devices/f79944e4-5a3d-11e8-99ce-479cbab002e4
+func getSysfsDev(sysfsDevStr string) (string, error) {
+	return filepath.EvalSymlinks(sysfsDevStr)
 }
 
 // BindDevicetoVFIO binds the device to vfio driver after unbinding from host.

virtcontainers/device/drivers/vfio_test.go

@@ -9,26 +9,38 @@ package drivers
 import (
 	"testing"
 
+	"github.com/kata-containers/runtime/virtcontainers/device/config"
 	"github.com/stretchr/testify/assert"
 )
 
-func TestGetBDF(t *testing.T) {
+func TestGetVFIODetails(t *testing.T) {
 	type testData struct {
 		deviceStr   string
-		expectedBDF string
+		expectedStr string
 	}
 
 	data := []testData{
 		{"0000:02:10.0", "02:10.0"},
 		{"0000:0210.0", ""},
+		{"f79944e4-5a3d-11e8-99ce-", ""},
+		{"f79944e4-5a3d-11e8-99ce", ""},
 		{"test", ""},
 		{"", ""},
 	}
 
 	for _, d := range data {
-		deviceBDF, err := getBDF(d.deviceStr)
-		assert.Equal(t, d.expectedBDF, deviceBDF)
-		if d.expectedBDF == "" {
+		deviceBDF, deviceSysfsDev, vfioDeviceType, err := getVFIODetails(d.deviceStr, "")
+
+		switch vfioDeviceType {
+		case config.VFIODeviceNormalType:
+			assert.Equal(t, d.expectedStr, deviceBDF)
+		case config.VFIODeviceMediatedType:
+			assert.Equal(t, d.expectedStr, deviceSysfsDev)
+		default:
+			assert.NotNil(t, err)
+		}
+
+		if d.expectedStr == "" {
 			assert.NotNil(t, err)
 		} else {
 			assert.Nil(t, err)

virtcontainers/qemu.go

@@ -795,16 +795,28 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) error {
 		// for pc machine type instead of bridge. This is useful for devices that require
 		// a large PCI BAR which is a currently a limitation with PCI bridges.
 		if q.state.HotplugVFIOOnRootBus {
-			return q.qmpMonitorCh.qmp.ExecuteVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF)
+			switch device.Type {
+			case config.VFIODeviceNormalType:
+				return q.qmpMonitorCh.qmp.ExecuteVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF)
+			case config.VFIODeviceMediatedType:
+				return q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, device.SysfsDev, "", "")
+			default:
+				return fmt.Errorf("Incorrect VFIO device type found")
+			}
 		}
 
 		addr, bridge, err := q.addDeviceToBridge(devID)
 		if err != nil {
 			return err
 		}
 
-		if err := q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, addr, bridge.ID); err != nil {
-			return err
+		switch device.Type {
+		case config.VFIODeviceNormalType:
+			return q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, addr, bridge.ID)
+		case config.VFIODeviceMediatedType:
+			return q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, device.SysfsDev, addr, bridge.ID)
+		default:
+			return fmt.Errorf("Incorrect VFIO device type found")
 		}
 	} else {
 		if !q.state.HotplugVFIOOnRootBus {