clh: Add vfio support

Add support to hotplug vfio devices. Use hypervisor API to attach devices via hotplug. Fixes: #2496 Signed-off-by: Jose Carlos Venegas Munoz <[email protected]>
kata-containers · Mar 19, 2020 · c518464 · c518464
1 parent aaa4e5c
commit c518464
Show file tree

Hide file tree

Showing 3 changed files with 41 additions and 4 deletions.
diff --git a/virtcontainers/clh.go b/virtcontainers/clh.go
@@ -50,8 +50,11 @@ const (
 const (
 	// Values are mandatory by http API
 	// Values based on:
-	clhTimeout            = 10
-	clhAPITimeout         = 1
+	clhTimeout    = 10
+	clhAPITimeout = 1
+	// Timeout for hot-plug - hotplug devices can take more time, than usual API calls
+	// Use longer time timeout for it.
+	clhHotPlugAPITimeout  = 5
 	clhStopSandboxTimeout = 3
 	clhSocket             = "clh.sock"
 	clhAPISocket          = "clh-api.sock"
@@ -76,6 +79,7 @@ type clhClient interface {
 	VmInfoGet(ctx context.Context) (chclient.VmInfo, *http.Response, error) //nolint:golint
 	BootVM(ctx context.Context) (*http.Response, error)
 	VmResizePut(ctx context.Context, vmResize chclient.VmResize) (*http.Response, error)
+	VmAddDevicePut(ctx context.Context, vmAddDevice chclient.VmAddDevice) (*http.Response, error)
 }
 
 type CloudHypervisorVersion struct {
@@ -372,9 +376,35 @@ func (clh *cloudHypervisor) getThreadIDs() (vcpuThreadIDs, error) {
 	return vcpuInfo, nil
 }
 
+func (clh *cloudHypervisor) hotPlugVFIODevice(device config.VFIODev) error {
+	cl := clh.client()
+	ctx, cancel := context.WithTimeout(context.Background(), clhHotPlugAPITimeout*time.Second)
+	defer cancel()
+
+	_, _, err := cl.VmmPingGet(ctx)
+	if err != nil {
+		return openAPIClientError(err)
+	}
+
+	_, err = cl.VmAddDevicePut(ctx, chclient.VmAddDevice{Path: device.SysfsDev})
+	if err != nil {
+		err = fmt.Errorf("Failed to hotplug device %+v %s", device, openAPIClientError(err))
+	}
+	return err
+}
+
 func (clh *cloudHypervisor) hotplugAddDevice(devInfo interface{}, devType deviceType) (interface{}, error) {
-	clh.Logger().WithField("function", "hotplugAddDevice").Warn("hotplug add device not supported")
-	return nil, nil
+	span, _ := clh.trace("hotplugAddDevice")
+	defer span.Finish()
+
+	switch devType {
+	case vfioDev:
+		device := devInfo.(*config.VFIODev)
+		return nil, clh.hotPlugVFIODevice(*device)
+	default:
+		return nil, fmt.Errorf("cannot hotplug device: unsupported device type '%v'", devType)
+	}
+
 }
 
 func (clh *cloudHypervisor) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error) {

diff --git a/virtcontainers/clh_test.go b/virtcontainers/clh_test.go
@@ -86,6 +86,11 @@ func (c *clhClientMock) VmResizePut(ctx context.Context, vmResize chclient.VmRes
 	return nil, nil
 }
 
+//nolint:golint
+func (c *clhClientMock) VmAddDevicePut(ctx context.Context, vmAddDevice chclient.VmAddDevice) (*http.Response, error) {
+	return nil, nil
+}
+
 func TestCloudHypervisorAddVSock(t *testing.T) {
 	assert := assert.New(t)
 	clh := cloudHypervisor{}

diff --git a/virtcontainers/device/drivers/vfio.go b/virtcontainers/device/drivers/vfio.go
@@ -194,6 +194,8 @@ func getVFIODetails(deviceFileName, iommuDevicesPath string) (deviceBDF, deviceS
 	case config.VFIODeviceNormalType:
 		// Get bdf of device eg. 0000:00:1c.0
 		deviceBDF = getBDF(deviceFileName)
+		// Get sysfs path used by cloud-hypervisor
+		deviceSysfsDev = filepath.Join(config.SysBusPciDevicesPath, deviceFileName)
 	case config.VFIODeviceMediatedType:
 		// Get sysfsdev of device eg. /sys/devices/pci0000:00/0000:00:02.0/f79944e4-5a3d-11e8-99ce-479cbab002e4
 		sysfsDevStr := filepath.Join(iommuDevicesPath, deviceFileName)