devices: hotplug vhost-user-blk device to qemu

vhost-user-blk device can be hotplugged or hotremoved
by QMP commands.

Fixes: #2380

Signed-off-by: Liu Xiaodong <[email protected]>

virtcontainers/device/api/interface.go

@@ -58,8 +58,9 @@ type Device interface {
 
 	// GetDeviceInfo returns device specific data used for hotplugging by hypervisor
 	// Caller could cast the return value to device specific struct
-	// e.g. Block device returns *config.BlockDrive and
-	// vfio device returns []*config.VFIODev
+	// e.g. Block device returns *config.BlockDrive,
+	// vfio device returns []*config.VFIODev,
+	// VhostUser device returns []*config.VhostUserDeviceAttrs
 	GetDeviceInfo() interface{}
 
 	// GetAttachCount returns how many times the device has been attached

virtcontainers/device/drivers/generic.go

@@ -14,7 +14,7 @@ import (
 	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 )
 
-// GenericDevice refers to a device that is neither a VFIO device or block device.
+// GenericDevice refers to a device that is neither a VFIO device, block device or VhostUserDevice.
 type GenericDevice struct {
 	ID         string
 	DeviceInfo *config.DeviceInfo

virtcontainers/device/drivers/vhost_user_blk.go

@@ -7,18 +7,27 @@
 package drivers
 
 import (
-	"encoding/hex"
-
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
 	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/runtime/virtcontainers/utils"
+	"github.com/sirupsen/logrus"
 )
 
 // VhostUserBlkDevice is a block vhost-user based device
 type VhostUserBlkDevice struct {
 	*GenericDevice
-	config.VhostUserDeviceAttrs
+	VhostUserDeviceAttrs *config.VhostUserDeviceAttrs
+}
+
+// NewVhostUserBlkDevice creates a new vhost-user block device based on DeviceInfo
+func NewVhostUserBlkDevice(devInfo *config.DeviceInfo) *VhostUserBlkDevice {
+	return &VhostUserBlkDevice{
+		GenericDevice: &GenericDevice{
+			ID:         devInfo.ID,
+			DeviceInfo: devInfo,
+		},
+	}
 }
 
 //
@@ -35,30 +44,103 @@ func (device *VhostUserBlkDevice) Attach(devReceiver api.DeviceReceiver) (err er
 	if skip {
 		return nil
 	}
+
+	// From the explanation of function attach in block.go, block index of
+	// a general block device is utilized for some situation.
+	// Since vhost-user-blk uses "vd" prefix in Linux kernel, not "sd",
+	// sandbox block index should be updated only if sandbox default block
+	// driver is "virtio-blk"/"virtio-blk-ccw"/"virtio-mmio" which uses
+	// "vd" prefix in Linux kernel.
+	index := -1
+	updateBlockIndex := isVirtioBlkBlockDriver(device.DeviceInfo.DriverOptions)
+	if updateBlockIndex {
+		index, err = devReceiver.GetAndSetSandboxBlockIndex()
+	}
+
 	defer func() {
 		if err != nil {
+			if updateBlockIndex {
+				devReceiver.UnsetSandboxBlockIndex(index)
+			}
 			device.bumpAttachCount(false)
 		}
 	}()
 
-	// generate a unique ID to be used for hypervisor commandline fields
-	randBytes, err := utils.GenerateRandomBytes(8)
 	if err != nil {
 		return err
 	}
-	id := hex.EncodeToString(randBytes)
 
-	device.DevID = id
-	device.Type = device.DeviceType()
+	vAttrs := &config.VhostUserDeviceAttrs{
+		DevID:      utils.MakeNameID("blk", device.DeviceInfo.ID, maxDevIDSize),
+		SocketPath: device.DeviceInfo.HostPath,
+		Type:       config.VhostUserBlk,
+		Index:      index,
+	}
+
+	deviceLogger().WithFields(logrus.Fields{
+		"device":     device.DeviceInfo.HostPath,
+		"SocketPath": vAttrs.SocketPath,
+		"Type":       config.VhostUserBlk,
+		"Index":      index,
+	}).Info("Attaching device")
+
+	device.VhostUserDeviceAttrs = vAttrs
+	if err = devReceiver.HotplugAddDevice(device, config.VhostUserBlk); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func isVirtioBlkBlockDriver(customOptions map[string]string) bool {
+	var blockDriverOption string
+
+	if customOptions == nil {
+		// User has not chosen a specific block device type
+		// Default to SCSI
+		blockDriverOption = "virtio-scsi"
+	} else {
+		blockDriverOption = customOptions["block-driver"]
+	}
+
+	if blockDriverOption == "virtio-blk" ||
+		blockDriverOption == "virtio-blk-ccw" ||
+		blockDriverOption == "virtio-mmio" {
+		return true
+	}
 
-	return devReceiver.AppendDevice(device)
+	return false
 }
 
 // Detach is standard interface of api.Device, it's used to remove device from some
 // DeviceReceiver
 func (device *VhostUserBlkDevice) Detach(devReceiver api.DeviceReceiver) error {
-	_, err := device.bumpAttachCount(false)
-	return err
+	skip, err := device.bumpAttachCount(false)
+	if err != nil {
+		return err
+	}
+	if skip {
+		return nil
+	}
+
+	defer func() {
+		if err != nil {
+			device.bumpAttachCount(true)
+		} else {
+			updateBlockIndex := isVirtioBlkBlockDriver(device.DeviceInfo.DriverOptions)
+			if updateBlockIndex {
+				devReceiver.UnsetSandboxBlockIndex(device.VhostUserDeviceAttrs.Index)
+			}
+		}
+	}()
+
+	deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Unplugging vhost-user-blk device")
+
+	if err = devReceiver.HotplugRemoveDevice(device, config.VhostUserBlk); err != nil {
+		deviceLogger().WithError(err).Error("Failed to unplug vhost-user-blk device")
+		return err
+	}
+	return nil
 }
 
 // DeviceType is standard interface of api.Device, it returns device type
@@ -68,19 +150,23 @@ func (device *VhostUserBlkDevice) DeviceType() config.DeviceType {
 
 // GetDeviceInfo returns device information used for creating
 func (device *VhostUserBlkDevice) GetDeviceInfo() interface{} {
-	device.Type = device.DeviceType()
-	return &device.VhostUserDeviceAttrs
+	return device.VhostUserDeviceAttrs
 }
 
 // Save converts Device to DeviceState
 func (device *VhostUserBlkDevice) Save() persistapi.DeviceState {
 	ds := device.GenericDevice.Save()
 	ds.Type = string(device.DeviceType())
-	ds.VhostUserDev = &persistapi.VhostUserDeviceAttrs{
-		DevID:      device.DevID,
-		SocketPath: device.SocketPath,
-		Type:       string(device.Type),
-		MacAddress: device.MacAddress,
+
+	vAttr := device.VhostUserDeviceAttrs
+	if vAttr != nil {
+		ds.VhostUserDev = &persistapi.VhostUserDeviceAttrs{
+			DevID:      vAttr.DevID,
+			SocketPath: vAttr.SocketPath,
+			Type:       string(vAttr.Type),
+			PCIAddr:    vAttr.PCIAddr,
+			Index:      vAttr.Index,
+		}
 	}
 	return ds
 }
@@ -95,11 +181,12 @@ func (device *VhostUserBlkDevice) Load(ds persistapi.DeviceState) {
 		return
 	}
 
-	device.VhostUserDeviceAttrs = config.VhostUserDeviceAttrs{
+	device.VhostUserDeviceAttrs = &config.VhostUserDeviceAttrs{
 		DevID:      dev.DevID,
 		SocketPath: dev.SocketPath,
 		Type:       config.DeviceType(dev.Type),
-		MacAddress: dev.MacAddress,
+		PCIAddr:    dev.PCIAddr,
+		Index:      dev.Index,
 	}
 }
 

virtcontainers/device/manager/manager.go

@@ -121,6 +121,12 @@ func (dm *deviceManager) createDevice(devInfo config.DeviceInfo) (dev api.Device
 	}
 	if isVFIO(path) {
 		return drivers.NewVFIODevice(&devInfo), nil
+	} else if isVhostUserBlk(devInfo) {
+		if devInfo.DriverOptions == nil {
+			devInfo.DriverOptions = make(map[string]string)
+		}
+		devInfo.DriverOptions["block-driver"] = dm.blockDriver
+		return drivers.NewVhostUserBlkDevice(&devInfo), nil
 	} else if isBlock(devInfo) {
 		if devInfo.DriverOptions == nil {
 			devInfo.DriverOptions = make(map[string]string)

virtcontainers/qemu.go

@@ -1145,6 +1145,40 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev
 	return nil
 }
 
+func (q *qemu) hotplugAddVhostUserBlkDevice(vAttr *config.VhostUserDeviceAttrs, op operation, devID string) (err error) {
+	err = q.qmpMonitorCh.qmp.ExecuteCharDevUnixSocketAdd(q.qmpMonitorCh.ctx, vAttr.DevID, vAttr.SocketPath, false, false)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		if err != nil {
+			q.qmpMonitorCh.qmp.ExecuteChardevDel(q.qmpMonitorCh.ctx, vAttr.DevID)
+		}
+	}()
+
+	driver := "vhost-user-blk-pci"
+	addr, bridge, err := q.arch.addDeviceToBridge(vAttr.DevID, types.PCI)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		if err != nil {
+			q.arch.removeDeviceFromBridge(vAttr.DevID)
+		}
+	}()
+
+	// PCI address is in the format bridge-addr/device-addr eg. "03/02"
+	vAttr.PCIAddr = fmt.Sprintf("%02x", bridge.Addr) + "/" + addr
+
+	if err = q.qmpMonitorCh.qmp.ExecutePCIVhostUserDevAdd(q.qmpMonitorCh.ctx, driver, devID, vAttr.DevID, addr, bridge.ID); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error {
 	err := q.qmpSetup()
 	if err != nil {
@@ -1174,6 +1208,38 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error
 	return err
 }
 
+func (q *qemu) hotplugVhostUserDevice(vAttr *config.VhostUserDeviceAttrs, op operation) error {
+	err := q.qmpSetup()
+	if err != nil {
+		return err
+	}
+
+	devID := "virtio-" + vAttr.DevID
+
+	if op == addDevice {
+		switch vAttr.Type {
+		case config.VhostUserBlk:
+			return q.hotplugAddVhostUserBlkDevice(vAttr, op, devID)
+		default:
+			return fmt.Errorf("Incorrect vhost-user device type found")
+		}
+	} else {
+		if err := q.arch.removeDeviceFromBridge(vAttr.DevID); err != nil {
+			return err
+		}
+
+		if err := q.qmpMonitorCh.qmp.ExecuteDeviceDel(q.qmpMonitorCh.ctx, devID); err != nil {
+			return err
+		}
+
+		if err := q.qmpMonitorCh.qmp.ExecuteChardevDel(q.qmpMonitorCh.ctx, vAttr.DevID); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err error) {
 	err = q.qmpSetup()
 	if err != nil {
@@ -1366,6 +1432,9 @@ func (q *qemu) hotplugDevice(devInfo interface{}, devType deviceType, op operati
 	case netDev:
 		device := devInfo.(Endpoint)
 		return nil, q.hotplugNetDevice(device, op)
+	case vhostuserDev:
+		vAttr := devInfo.(*config.VhostUserDeviceAttrs)
+		return nil, q.hotplugVhostUserDevice(vAttr, op)
 	default:
 		return nil, fmt.Errorf("cannot hotplug device: unsupported device type '%v'", devType)
 	}

virtcontainers/qemu_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/runtime/virtcontainers/persist"
 	"github.com/kata-containers/runtime/virtcontainers/types"
+	"github.com/kata-containers/runtime/virtcontainers/utils"
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 )
@@ -221,6 +222,27 @@ func TestQemuAddDeviceFsDev(t *testing.T) {
 	testQemuAddDevice(t, volume, fsDev, expectedOut)
 }
 
+func TestQemuAddDeviceVhostUserBlk(t *testing.T) {
+	socketPath := "/test/socket/path"
+	devID := "testDevID"
+
+	expectedOut := []govmmQemu.Device{
+		govmmQemu.VhostUserDevice{
+			SocketPath:    socketPath,
+			CharDevID:     utils.MakeNameID("char", devID, maxDevIDSize),
+			VhostUserType: govmmQemu.VhostUserBlk,
+		},
+	}
+
+	vDevice := config.VhostUserDeviceAttrs{
+		DevID:      devID,
+		SocketPath: socketPath,
+		Type:       config.VhostUserBlk,
+	}
+
+	testQemuAddDevice(t, vDevice, vhostuserDev, expectedOut)
+}
+
 func TestQemuAddDeviceSerialPortDev(t *testing.T) {
 	deviceID := "channelTest"
 	id := "charchTest"

virtcontainers/sandbox.go

@@ -1662,6 +1662,13 @@ func (s *Sandbox) HotplugAddDevice(device api.Device, devType config.DeviceType)
 		}
 		_, err := s.hypervisor.hotplugAddDevice(blockDevice.BlockDrive, blockDev)
 		return err
+	case config.VhostUserBlk:
+		vhostUserBlkDevice, ok := device.(*drivers.VhostUserBlkDevice)
+		if !ok {
+			return fmt.Errorf("device type mismatch, expect device type to be %s", devType)
+		}
+		_, err := s.hypervisor.hotplugAddDevice(vhostUserBlkDevice.VhostUserDeviceAttrs, vhostuserDev)
+		return err
 	case config.DeviceGeneric:
 		// TODO: what?
 		return nil
@@ -1699,6 +1706,13 @@ func (s *Sandbox) HotplugRemoveDevice(device api.Device, devType config.DeviceTy
 		}
 		_, err := s.hypervisor.hotplugRemoveDevice(blockDrive, blockDev)
 		return err
+	case config.VhostUserBlk:
+		vhostUserDeviceAttrs, ok := device.GetDeviceInfo().(*config.VhostUserDeviceAttrs)
+		if !ok {
+			return fmt.Errorf("device type mismatch, expect device type to be %s", devType)
+		}
+		_, err := s.hypervisor.hotplugRemoveDevice(vhostUserDeviceAttrs, vhostuserDev)
+		return err
 	case config.DeviceGeneric:
 		// TODO: what?
 		return nil