diff --git a/virtcontainers/api.go b/virtcontainers/api.go index 42930cd81f..72687bda61 100644 --- a/virtcontainers/api.go +++ b/virtcontainers/api.go @@ -10,6 +10,7 @@ import ( "runtime" "syscall" + deviceApi "github.com/kata-containers/runtime/virtcontainers/device/api" "github.com/sirupsen/logrus" ) @@ -27,6 +28,7 @@ func SetLogger(logger logrus.FieldLogger) { } virtLog = logger.WithFields(fields) + deviceApi.SetLogger(virtLog) } // CreateSandbox is the virtcontainers sandbox creation entry point. diff --git a/virtcontainers/container.go b/virtcontainers/container.go index 363b923d6c..c4574fe5e4 100644 --- a/virtcontainers/container.go +++ b/virtcontainers/container.go @@ -16,6 +16,11 @@ import ( "github.com/sirupsen/logrus" "golang.org/x/sys/unix" + + "github.com/kata-containers/runtime/virtcontainers/device/api" + "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/device/drivers" + "github.com/kata-containers/runtime/virtcontainers/utils" ) // Process gathers data related to a container process. @@ -83,7 +88,7 @@ type ContainerConfig struct { Mounts []Mount // Device configuration for devices that must be available within the container. - DeviceInfos []DeviceInfo + DeviceInfos []config.DeviceInfo // Resources container resources Resources ContainerResources @@ -134,7 +139,7 @@ type Container struct { mounts []Mount - devices []Device + devices []api.Device systemMountsInfo SystemMountsInfo } @@ -245,7 +250,7 @@ func (c *Container) storeDevices() error { return c.sandbox.storage.storeContainerDevices(c.sandboxID, c.id, c.devices) } -func (c *Container) fetchDevices() ([]Device, error) { +func (c *Container) fetchDevices() ([]api.Device, error) { return c.sandbox.storage.fetchContainerDevices(c.sandboxID, c.id) } @@ -321,9 +326,10 @@ func (c *Container) mountSharedDirMounts(hostSharedDir, guestSharedDir string) ( // Check if mount is a block device file. If it is, the block device will be attached to the host // instead of passing this as a shared mount. if c.checkBlockDeviceSupport() && stat.Mode&unix.S_IFBLK == unix.S_IFBLK { - b := &BlockDevice{ - DeviceType: DeviceBlock, - DeviceInfo: DeviceInfo{ + // TODO: remove dependency of package drivers + b := &drivers.BlockDevice{ + DevType: config.DeviceBlock, + DeviceInfo: config.DeviceInfo{ HostPath: m.Source, ContainerPath: m.Destination, DevType: "b", @@ -331,7 +337,7 @@ func (c *Container) mountSharedDirMounts(hostSharedDir, guestSharedDir string) ( } // Attach this block device, all other devices passed in the config have been attached at this point - if err := b.attach(c.sandbox.hypervisor, c); err != nil { + if err := b.Attach(c.sandbox); err != nil { return nil, err } @@ -346,7 +352,7 @@ func (c *Container) mountSharedDirMounts(hostSharedDir, guestSharedDir string) ( continue } - randBytes, err := generateRandomBytes(8) + randBytes, err := utils.GenerateRandomBytes(8) if err != nil { return nil, err } @@ -449,7 +455,7 @@ func newContainer(sandbox *Sandbox, contConfig ContainerConfig) (*Container, err // If devices were not found in storage, create Device implementations // from the configuration. This should happen at create. - devices, err := newDevices(contConfig.DeviceInfos) + devices, err := sandbox.devManager.NewDevices(contConfig.DeviceInfos) if err != nil { return &Container{}, err } @@ -824,8 +830,8 @@ func (c *Container) hotplugDrive() error { } // Add drive with id as container id - devID := makeNameID("drive", c.id) - drive := Drive{ + devID := utils.MakeNameID("drive", c.id, maxDevIDSize) + drive := drivers.Drive{ File: devicePath, Format: "raw", ID: devID, @@ -861,8 +867,8 @@ func (c *Container) removeDrive() (err error) { if c.isDriveUsed() && c.state.HotpluggedDrive { c.Logger().Info("unplugging block device") - devID := makeNameID("drive", c.id) - drive := &Drive{ + devID := utils.MakeNameID("drive", c.id, maxDevIDSize) + drive := &drivers.Drive{ ID: devID, } @@ -880,7 +886,7 @@ func (c *Container) removeDrive() (err error) { func (c *Container) attachDevices() error { for _, device := range c.devices { - if err := device.attach(c.sandbox.hypervisor, c); err != nil { + if err := device.Attach(c.sandbox); err != nil { return err } } @@ -890,7 +896,7 @@ func (c *Container) attachDevices() error { func (c *Container) detachDevices() error { for _, device := range c.devices { - if err := device.detach(c.sandbox.hypervisor); err != nil { + if err := device.Detach(c.sandbox); err != nil { return err } } @@ -904,7 +910,7 @@ func (c *Container) addResources() error { return nil } - vCPUs := ConstraintsToVCPUs(c.config.Resources.CPUQuota, c.config.Resources.CPUPeriod) + vCPUs := utils.ConstraintsToVCPUs(c.config.Resources.CPUQuota, c.config.Resources.CPUPeriod) if vCPUs != 0 { virtLog.Debugf("hot adding %d vCPUs", vCPUs) if err := c.sandbox.hypervisor.hotplugAddDevice(uint32(vCPUs), cpuDev); err != nil { @@ -923,7 +929,7 @@ func (c *Container) removeResources() error { return nil } - vCPUs := ConstraintsToVCPUs(c.config.Resources.CPUQuota, c.config.Resources.CPUPeriod) + vCPUs := utils.ConstraintsToVCPUs(c.config.Resources.CPUQuota, c.config.Resources.CPUPeriod) if vCPUs != 0 { virtLog.Debugf("hot removing %d vCPUs", vCPUs) if err := c.sandbox.hypervisor.hotplugRemoveDevice(uint32(vCPUs), cpuDev); err != nil { diff --git a/virtcontainers/device.go b/virtcontainers/device.go deleted file mode 100644 index 6b852bcbbb..0000000000 --- a/virtcontainers/device.go +++ /dev/null @@ -1,644 +0,0 @@ -// Copyright (c) 2017 Intel Corporation -// -// SPDX-License-Identifier: Apache-2.0 -// - -package virtcontainers - -import ( - "encoding/hex" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "strconv" - "strings" - - "github.com/go-ini/ini" - "github.com/sirupsen/logrus" -) - -const ( - // DeviceVFIO is the VFIO device type - DeviceVFIO = "vfio" - - // DeviceBlock is the block device type - DeviceBlock = "block" - - // DeviceGeneric is a generic device type - DeviceGeneric = "generic" -) - -// Defining this as a variable instead of a const, to allow -// overriding this in the tests. -var sysIOMMUPath = "/sys/kernel/iommu_groups" - -var sysDevPrefix = "/sys/dev" - -const ( - vfioPath = "/dev/vfio/" -) - -// Device is the virtcontainers device interface. -type Device interface { - attach(hypervisor, *Container) error - detach(hypervisor) error - deviceType() string -} - -// DeviceInfo is an embedded type that contains device data common to all types of devices. -type DeviceInfo struct { - // Device path on host - HostPath string - - // Device path inside the container - ContainerPath string - - // Type of device: c, b, u or p - // c , u - character(unbuffered) - // p - FIFO - // b - block(buffered) special file - // More info in mknod(1). - DevType string - - // Major, minor numbers for device. - Major int64 - Minor int64 - - // FileMode permission bits for the device. - FileMode os.FileMode - - // id of the device owner. - UID uint32 - - // id of the device group. - GID uint32 - - // Hotplugged is used to store device state indicating if the - // device was hotplugged. - Hotplugged bool - - // ID for the device that is passed to the hypervisor. - ID string -} - -func deviceLogger() *logrus.Entry { - return virtLog.WithField("subsystem", "device") -} - -// VFIODevice is a vfio device meant to be passed to the hypervisor -// to be used by the Virtual Machine. -type VFIODevice struct { - DeviceType string - DeviceInfo DeviceInfo - BDF string -} - -func newVFIODevice(devInfo DeviceInfo) *VFIODevice { - return &VFIODevice{ - DeviceType: DeviceVFIO, - DeviceInfo: devInfo, - } -} - -func (device *VFIODevice) attach(h hypervisor, c *Container) error { - vfioGroup := filepath.Base(device.DeviceInfo.HostPath) - iommuDevicesPath := filepath.Join(sysIOMMUPath, vfioGroup, "devices") - - deviceFiles, err := ioutil.ReadDir(iommuDevicesPath) - if err != nil { - return err - } - - // Pass all devices in iommu group - for _, deviceFile := range deviceFiles { - - //Get bdf of device eg 0000:00:1c.0 - deviceBDF, err := getBDF(deviceFile.Name()) - if err != nil { - return err - } - - device.BDF = deviceBDF - - randBytes, err := generateRandomBytes(8) - if err != nil { - return err - } - device.DeviceInfo.ID = hex.EncodeToString(randBytes) - - if err := h.hotplugAddDevice(*device, vfioDev); err != nil { - deviceLogger().WithError(err).Error("Failed to add device") - return err - } - - deviceLogger().WithFields(logrus.Fields{ - "device-group": device.DeviceInfo.HostPath, - "device-type": "vfio-passthrough", - }).Info("Device group attached") - } - - return nil -} - -func (device *VFIODevice) detach(h hypervisor) error { - return nil -} - -func (device *VFIODevice) deviceType() string { - return device.DeviceType -} - -// VhostUserDeviceType - represents a vhost-user device type -// Currently support just VhostUserNet -type VhostUserDeviceType string - -const ( - //VhostUserSCSI - SCSI based vhost-user type - VhostUserSCSI = "vhost-user-scsi-pci" - //VhostUserNet - net based vhost-user type - VhostUserNet = "virtio-net-pci" - //VhostUserBlk represents a block vhostuser device type - VhostUserBlk = "vhost-user-blk-pci" -) - -// VhostUserDevice represents a vhost-user device. Shared -// attributes of a vhost-user device can be retrieved using -// the Attrs() method. Unique data can be obtained by casting -// the object to the proper type. -type VhostUserDevice interface { - Attrs() *VhostUserDeviceAttrs - Type() string -} - -// VhostUserDeviceAttrs represents data shared by most vhost-user devices -type VhostUserDeviceAttrs struct { - DeviceType string - DeviceInfo DeviceInfo - SocketPath string - ID string -} - -// VhostUserNetDevice is a network vhost-user based device -type VhostUserNetDevice struct { - VhostUserDeviceAttrs - MacAddress string -} - -// Attrs returns the VhostUserDeviceAttrs associated with the vhost-user device -func (vhostUserNetDevice *VhostUserNetDevice) Attrs() *VhostUserDeviceAttrs { - return &vhostUserNetDevice.VhostUserDeviceAttrs -} - -// Type returns the type associated with the vhost-user device -func (vhostUserNetDevice *VhostUserNetDevice) Type() string { - return VhostUserNet -} - -// VhostUserSCSIDevice is a SCSI vhost-user based device -type VhostUserSCSIDevice struct { - VhostUserDeviceAttrs -} - -// Attrs returns the VhostUserDeviceAttrs associated with the vhost-user device -func (vhostUserSCSIDevice *VhostUserSCSIDevice) Attrs() *VhostUserDeviceAttrs { - return &vhostUserSCSIDevice.VhostUserDeviceAttrs -} - -// Type returns the type associated with the vhost-user device -func (vhostUserSCSIDevice *VhostUserSCSIDevice) Type() string { - return VhostUserSCSI -} - -// VhostUserBlkDevice is a block vhost-user based device -type VhostUserBlkDevice struct { - VhostUserDeviceAttrs -} - -// Attrs returns the VhostUserDeviceAttrs associated with the vhost-user device -func (vhostUserBlkDevice *VhostUserBlkDevice) Attrs() *VhostUserDeviceAttrs { - return &vhostUserBlkDevice.VhostUserDeviceAttrs -} - -// Type returns the type associated with the vhost-user device -func (vhostUserBlkDevice *VhostUserBlkDevice) Type() string { - return VhostUserBlk -} - -// vhostUserAttach handles the common logic among all of the vhost-user device's -// attach functions -func vhostUserAttach(device VhostUserDevice, h hypervisor, c *Container) (err error) { - // generate a unique ID to be used for hypervisor commandline fields - randBytes, err := generateRandomBytes(8) - if err != nil { - return err - } - id := hex.EncodeToString(randBytes) - - device.Attrs().ID = id - - return h.addDevice(device, vhostuserDev) -} - -// -// VhostUserNetDevice's implementation of the device interface: -// -func (vhostUserNetDevice *VhostUserNetDevice) attach(h hypervisor, c *Container) (err error) { - return vhostUserAttach(vhostUserNetDevice, h, c) -} - -func (vhostUserNetDevice *VhostUserNetDevice) detach(h hypervisor) error { - return nil -} - -func (vhostUserNetDevice *VhostUserNetDevice) deviceType() string { - return vhostUserNetDevice.DeviceType -} - -// -// VhostUserBlkDevice's implementation of the device interface: -// -func (vhostUserBlkDevice *VhostUserBlkDevice) attach(h hypervisor, c *Container) (err error) { - return vhostUserAttach(vhostUserBlkDevice, h, c) -} - -func (vhostUserBlkDevice *VhostUserBlkDevice) detach(h hypervisor) error { - return nil -} - -func (vhostUserBlkDevice *VhostUserBlkDevice) deviceType() string { - return vhostUserBlkDevice.DeviceType -} - -// -// VhostUserSCSIDevice's implementation of the device interface: -// -func (vhostUserSCSIDevice *VhostUserSCSIDevice) attach(h hypervisor, c *Container) (err error) { - return vhostUserAttach(vhostUserSCSIDevice, h, c) -} - -func (vhostUserSCSIDevice *VhostUserSCSIDevice) detach(h hypervisor) error { - return nil -} - -func (vhostUserSCSIDevice *VhostUserSCSIDevice) deviceType() string { - return vhostUserSCSIDevice.DeviceType -} - -// Long term, this should be made more configurable. For now matching path -// provided by CNM VPP and OVS-DPDK plugins, available at github.com/clearcontainers/vpp and -// github.com/clearcontainers/ovsdpdk. The plugins create the socket on the host system -// using this path. -const hostSocketSearchPath = "/tmp/vhostuser_%s/vhu.sock" - -// findVhostUserNetSocketPath checks if an interface is a dummy placeholder -// for a vhost-user socket, and if it is it returns the path to the socket -func findVhostUserNetSocketPath(netInfo NetworkInfo) (string, error) { - if netInfo.Iface.Name == "lo" { - return "", nil - } - - // check for socket file existence at known location. - for _, addr := range netInfo.Addrs { - socketPath := fmt.Sprintf(hostSocketSearchPath, addr.IPNet.IP) - if _, err := os.Stat(socketPath); err == nil { - return socketPath, nil - } - } - - return "", nil -} - -// vhostUserSocketPath returns the path of the socket discovered. This discovery -// will vary depending on the type of vhost-user socket. -// Today only VhostUserNetDevice is supported. -func vhostUserSocketPath(info interface{}) (string, error) { - - switch v := info.(type) { - case NetworkInfo: - return findVhostUserNetSocketPath(v) - default: - return "", nil - } - -} - -// BlockDevice refers to a block storage device implementation. -type BlockDevice struct { - DeviceType string - DeviceInfo DeviceInfo - - // SCSI Address of the block device, in case the device is attached using SCSI driver - // SCSI address is in the format SCSI-Id:LUN - SCSIAddr string - - // Path at which the device appears inside the VM, outside of the container mount namespace - VirtPath string - - // PCI Slot of the block device - PCIAddr string -} - -func newBlockDevice(devInfo DeviceInfo) *BlockDevice { - return &BlockDevice{ - DeviceType: DeviceBlock, - DeviceInfo: devInfo, - } -} - -func (device *BlockDevice) attach(h hypervisor, c *Container) (err error) { - randBytes, err := generateRandomBytes(8) - if err != nil { - return err - } - - device.DeviceInfo.ID = hex.EncodeToString(randBytes) - - // Increment the block index for the sandbox. This is used to determine the name - // for the block device in the case where the block device is used as container - // rootfs and the predicted block device name needs to be provided to the agent. - index, err := c.sandbox.getAndSetSandboxBlockIndex() - - defer func() { - if err != nil { - c.sandbox.decrementSandboxBlockIndex() - } - }() - - if err != nil { - return err - } - - drive := Drive{ - File: device.DeviceInfo.HostPath, - Format: "raw", - ID: makeNameID("drive", device.DeviceInfo.ID), - Index: index, - } - - driveName, err := getVirtDriveName(index) - if err != nil { - return err - } - - deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Attaching block device") - - if err = h.hotplugAddDevice(&drive, blockDev); err != nil { - return err - } - - device.DeviceInfo.Hotplugged = true - - if c.sandbox.config.HypervisorConfig.BlockDeviceDriver == VirtioBlock { - device.VirtPath = filepath.Join("/dev", driveName) - device.PCIAddr = drive.PCIAddr - } else { - scsiAddr, err := getSCSIAddress(index) - if err != nil { - return err - } - - device.SCSIAddr = scsiAddr - } - - return nil -} - -func (device BlockDevice) detach(h hypervisor) error { - if device.DeviceInfo.Hotplugged { - deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Unplugging block device") - - drive := &Drive{ - ID: makeNameID("drive", device.DeviceInfo.ID), - } - - if err := h.hotplugRemoveDevice(drive, blockDev); err != nil { - deviceLogger().WithError(err).Error("Failed to unplug block device") - return err - } - - } - return nil -} - -func (device *BlockDevice) deviceType() string { - return device.DeviceType -} - -// GenericDevice refers to a device that is neither a VFIO device or block device. -type GenericDevice struct { - DeviceType string - DeviceInfo DeviceInfo -} - -func newGenericDevice(devInfo DeviceInfo) *GenericDevice { - return &GenericDevice{ - DeviceType: DeviceGeneric, - DeviceInfo: devInfo, - } -} - -func (device *GenericDevice) attach(h hypervisor, c *Container) error { - return nil -} - -func (device *GenericDevice) detach(h hypervisor) error { - return nil -} - -func (device *GenericDevice) deviceType() string { - return device.DeviceType -} - -// isVFIO checks if the device provided is a vfio group. -func isVFIO(hostPath string) bool { - // Ignore /dev/vfio/vfio character device - if strings.HasPrefix(hostPath, filepath.Join(vfioPath, "vfio")) { - return false - } - - if strings.HasPrefix(hostPath, vfioPath) && len(hostPath) > len(vfioPath) { - return true - } - - return false -} - -// isBlock checks if the device is a block device. -func isBlock(devInfo DeviceInfo) bool { - if devInfo.DevType == "b" { - return true - } - - return false -} - -func createDevice(devInfo DeviceInfo) Device { - path := devInfo.HostPath - - if isVFIO(path) { - return newVFIODevice(devInfo) - } else if isBlock(devInfo) { - return newBlockDevice(devInfo) - } else { - deviceLogger().WithField("device", path).Info("Device has not been passed to the container") - return newGenericDevice(devInfo) - } -} - -// GetHostPath is used to fetcg the host path for the device. -// The path passed in the spec refers to the path that should appear inside the container. -// We need to find the actual device path on the host based on the major-minor numbers of the device. -func GetHostPath(devInfo DeviceInfo) (string, error) { - if devInfo.ContainerPath == "" { - return "", fmt.Errorf("Empty path provided for device") - } - - var pathComp string - - switch devInfo.DevType { - case "c", "u": - pathComp = "char" - case "b": - pathComp = "block" - default: - // Unsupported device types. Return nil error to ignore devices - // that cannot be handled currently. - return "", nil - } - - format := strconv.FormatInt(devInfo.Major, 10) + ":" + strconv.FormatInt(devInfo.Minor, 10) - sysDevPath := filepath.Join(sysDevPrefix, pathComp, format, "uevent") - - if _, err := os.Stat(sysDevPath); err != nil { - // Some devices(eg. /dev/fuse, /dev/cuse) do not always implement sysfs interface under /sys/dev - // These devices are passed by default by docker. - // - // Simply return the path passed in the device configuration, this does mean that no device renames are - // supported for these devices. - - if os.IsNotExist(err) { - return devInfo.ContainerPath, nil - } - - return "", err - } - - content, err := ini.Load(sysDevPath) - if err != nil { - return "", err - } - - devName, err := content.Section("").GetKey("DEVNAME") - if err != nil { - return "", err - } - - return filepath.Join("/dev", devName.String()), nil -} - -// GetHostPathFunc is function pointer used to mock GetHostPath in tests. -var GetHostPathFunc = GetHostPath - -func newDevices(devInfos []DeviceInfo) ([]Device, error) { - var devices []Device - - for _, devInfo := range devInfos { - hostPath, err := GetHostPathFunc(devInfo) - if err != nil { - return nil, err - } - - devInfo.HostPath = hostPath - device := createDevice(devInfo) - devices = append(devices, device) - } - - return devices, nil -} - -// getBDF returns the BDF of pci device -// Expected input strng format is []:[][].[] eg. 0000:02:10.0 -func getBDF(deviceSysStr string) (string, error) { - tokens := strings.Split(deviceSysStr, ":") - - if len(tokens) != 3 { - return "", fmt.Errorf("Incorrect number of tokens found while parsing bdf for device : %s", deviceSysStr) - } - - tokens = strings.SplitN(deviceSysStr, ":", 2) - return tokens[1], nil -} - -// bind/unbind paths to aid in SRIOV VF bring-up/restore -var pciDriverUnbindPath = "/sys/bus/pci/devices/%s/driver/unbind" -var pciDriverBindPath = "/sys/bus/pci/drivers/%s/bind" -var vfioRemoveIDPath = "/sys/bus/pci/drivers/vfio-pci/remove_id" -var vfioNewIDPath = "/sys/bus/pci/drivers/vfio-pci/new_id" - -// bindDevicetoVFIO binds the device to vfio driver after unbinding from host. -// Will be called by a network interface or a generic pcie device. -func bindDevicetoVFIO(bdf, hostDriver, vendorDeviceID string) error { - - // Unbind from the host driver - unbindDriverPath := fmt.Sprintf(pciDriverUnbindPath, bdf) - deviceLogger().WithFields(logrus.Fields{ - "device-bdf": bdf, - "driver-path": unbindDriverPath, - }).Info("Unbinding device from driver") - - if err := writeToFile(unbindDriverPath, []byte(bdf)); err != nil { - return err - } - - // Add device id to vfio driver. - deviceLogger().WithFields(logrus.Fields{ - "vendor-device-id": vendorDeviceID, - "vfio-new-id-path": vfioNewIDPath, - }).Info("Writing vendor-device-id to vfio new-id path") - - if err := writeToFile(vfioNewIDPath, []byte(vendorDeviceID)); err != nil { - return err - } - - // Bind to vfio-pci driver. - bindDriverPath := fmt.Sprintf(pciDriverBindPath, "vfio-pci") - - deviceLogger().WithFields(logrus.Fields{ - "device-bdf": bdf, - "driver-path": bindDriverPath, - }).Info("Binding device to vfio driver") - - // Device may be already bound at this time because of earlier write to new_id, ignore error - writeToFile(bindDriverPath, []byte(bdf)) - - return nil -} - -// bindDevicetoHost binds the device to the host driver driver after unbinding from vfio-pci. -func bindDevicetoHost(bdf, hostDriver, vendorDeviceID string) error { - // Unbind from vfio-pci driver - unbindDriverPath := fmt.Sprintf(pciDriverUnbindPath, bdf) - deviceLogger().WithFields(logrus.Fields{ - "device-bdf": bdf, - "driver-path": unbindDriverPath, - }).Info("Unbinding device from driver") - - if err := writeToFile(unbindDriverPath, []byte(bdf)); err != nil { - return err - } - - // To prevent new VFs from binding to VFIO-PCI, remove_id - if err := writeToFile(vfioRemoveIDPath, []byte(vendorDeviceID)); err != nil { - return err - } - - // Bind back to host driver - bindDriverPath := fmt.Sprintf(pciDriverBindPath, hostDriver) - deviceLogger().WithFields(logrus.Fields{ - "device-bdf": bdf, - "driver-path": bindDriverPath, - }).Info("Binding back device to host driver") - - return writeToFile(bindDriverPath, []byte(bdf)) -} diff --git a/virtcontainers/device/api/interface.go b/virtcontainers/device/api/interface.go new file mode 100644 index 0000000000..d6a4072d33 --- /dev/null +++ b/virtcontainers/device/api/interface.go @@ -0,0 +1,61 @@ +// Copyright (c) 2017-2018 Intel Corporation +// Copyright (c) 2018 Huawei Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package api + +import ( + "github.com/sirupsen/logrus" + + "github.com/kata-containers/runtime/virtcontainers/device/config" +) + +var devLogger = logrus.FieldLogger(logrus.New()) + +// SetLogger sets the logger for device api package. +func SetLogger(logger logrus.FieldLogger) { + devLogger = logger +} + +// DeviceLogger returns logger for device management +func DeviceLogger() *logrus.Entry { + return devLogger.WithField("subsystem", "device") +} + +// DeviceReceiver is an interface used for accepting devices +// a device should be attached/added/plugged to a DeviceReceiver +type DeviceReceiver interface { + HotplugAddDevice(Device, config.DeviceType) error + HotplugRemoveDevice(Device, config.DeviceType) error + + // this is only for virtio-blk and virtio-scsi support + GetAndSetSandboxBlockIndex() (int, error) + DecrementSandboxBlockIndex() error + + // this is for vhost_user devices + AddVhostUserDevice(VhostUserDevice, config.DeviceType) error +} + +// VhostUserDevice represents a vhost-user device. Shared +// attributes of a vhost-user device can be retrieved using +// the Attrs() method. Unique data can be obtained by casting +// the object to the proper type. +type VhostUserDevice interface { + Attrs() *config.VhostUserDeviceAttrs + Type() config.DeviceType +} + +// Device is the virtcontainers device interface. +type Device interface { + Attach(DeviceReceiver) error + Detach(DeviceReceiver) error + DeviceType() config.DeviceType +} + +// DeviceManager can be used to create a new device, this can be used as single +// device management object. +type DeviceManager interface { + NewDevices(devInfos []config.DeviceInfo) ([]Device, error) +} diff --git a/virtcontainers/device/api/mockDeviceReceiver.go b/virtcontainers/device/api/mockDeviceReceiver.go new file mode 100644 index 0000000000..b5d2472c3c --- /dev/null +++ b/virtcontainers/device/api/mockDeviceReceiver.go @@ -0,0 +1,38 @@ +// Copyright (c) 2018 Huawei Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package api + +import ( + "github.com/kata-containers/runtime/virtcontainers/device/config" +) + +// MockDeviceReceiver is a fake DeviceReceiver API implementation only used for test +type MockDeviceReceiver struct{} + +// HotplugAddDevice adds a new device +func (mockDC *MockDeviceReceiver) HotplugAddDevice(Device, config.DeviceType) error { + return nil +} + +// HotplugRemoveDevice removes a device +func (mockDC *MockDeviceReceiver) HotplugRemoveDevice(Device, config.DeviceType) error { + return nil +} + +// GetAndSetSandboxBlockIndex is used for get and set virtio-blk indexes +func (mockDC *MockDeviceReceiver) GetAndSetSandboxBlockIndex() (int, error) { + return 0, nil +} + +// DecrementSandboxBlockIndex decreases virtio-blk index by one +func (mockDC *MockDeviceReceiver) DecrementSandboxBlockIndex() error { + return nil +} + +// AddVhostUserDevice adds new vhost user device +func (mockDC *MockDeviceReceiver) AddVhostUserDevice(VhostUserDevice, config.DeviceType) error { + return nil +} diff --git a/virtcontainers/device/config/config.go b/virtcontainers/device/config/config.go new file mode 100644 index 0000000000..1ab83ddc91 --- /dev/null +++ b/virtcontainers/device/config/config.go @@ -0,0 +1,150 @@ +// Copyright (c) 2017-2018 Intel Corporation +// Copyright (c) 2018 Huawei Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package config + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + + "github.com/go-ini/ini" +) + +// DeviceType indicates device type +type DeviceType string + +const ( + // DeviceVFIO is the VFIO device type + DeviceVFIO DeviceType = "vfio" + + // DeviceBlock is the block device type + DeviceBlock DeviceType = "block" + + // DeviceGeneric is a generic device type + DeviceGeneric DeviceType = "generic" + + //VhostUserSCSI - SCSI based vhost-user type + VhostUserSCSI = "vhost-user-scsi-pci" + + //VhostUserNet - net based vhost-user type + VhostUserNet = "virtio-net-pci" + + //VhostUserBlk represents a block vhostuser device type + VhostUserBlk = "vhost-user-blk-pci" +) + +// Defining these as a variable instead of a const, to allow +// overriding this in the tests. + +// SysDevPrefix is static string of /sys/dev +var SysDevPrefix = "/sys/dev" + +// SysIOMMUPath is static string of /sys/kernel/iommu_groups +var SysIOMMUPath = "/sys/kernel/iommu_groups" + +// DeviceInfo is an embedded type that contains device data common to all types of devices. +type DeviceInfo struct { + // Device path on host + HostPath string + + // Device path inside the container + ContainerPath string + + // Type of device: c, b, u or p + // c , u - character(unbuffered) + // p - FIFO + // b - block(buffered) special file + // More info in mknod(1). + DevType string + + // Major, minor numbers for device. + Major int64 + Minor int64 + + // FileMode permission bits for the device. + FileMode os.FileMode + + // id of the device owner. + UID uint32 + + // id of the device group. + GID uint32 + + // Hotplugged is used to store device state indicating if the + // device was hotplugged. + Hotplugged bool + + // ID for the device that is passed to the hypervisor. + ID string + + // DriverOptions is specific options for each device driver + // for example, for BlockDevice, we can set DriverOptions["blockDriver"]="virtio-blk" + DriverOptions map[string]string +} + +// VhostUserDeviceAttrs represents data shared by most vhost-user devices +type VhostUserDeviceAttrs struct { + DevType DeviceType + DeviceInfo DeviceInfo + SocketPath string + ID string +} + +// GetHostPathFunc is function pointer used to mock GetHostPath in tests. +var GetHostPathFunc = GetHostPath + +// GetHostPath is used to fetch the host path for the device. +// The path passed in the spec refers to the path that should appear inside the container. +// We need to find the actual device path on the host based on the major-minor numbers of the device. +func GetHostPath(devInfo DeviceInfo) (string, error) { + if devInfo.ContainerPath == "" { + return "", fmt.Errorf("Empty path provided for device") + } + + var pathComp string + + switch devInfo.DevType { + case "c", "u": + pathComp = "char" + case "b": + pathComp = "block" + default: + // Unsupported device types. Return nil error to ignore devices + // that cannot be handled currently. + return "", nil + } + + format := strconv.FormatInt(devInfo.Major, 10) + ":" + strconv.FormatInt(devInfo.Minor, 10) + sysDevPath := filepath.Join(SysDevPrefix, pathComp, format, "uevent") + + if _, err := os.Stat(sysDevPath); err != nil { + // Some devices(eg. /dev/fuse, /dev/cuse) do not always implement sysfs interface under /sys/dev + // These devices are passed by default by docker. + // + // Simply return the path passed in the device configuration, this does mean that no device renames are + // supported for these devices. + + if os.IsNotExist(err) { + return devInfo.ContainerPath, nil + } + + return "", err + } + + content, err := ini.Load(sysDevPath) + if err != nil { + return "", err + } + + devName, err := content.Section("").GetKey("DEVNAME") + if err != nil { + return "", err + } + + return filepath.Join("/dev", devName.String()), nil +} diff --git a/virtcontainers/device/drivers/block.go b/virtcontainers/device/drivers/block.go new file mode 100644 index 0000000000..175dc8cea2 --- /dev/null +++ b/virtcontainers/device/drivers/block.go @@ -0,0 +1,145 @@ +// Copyright (c) 2017-2018 Intel Corporation +// Copyright (c) 2018 Huawei Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package drivers + +import ( + "encoding/hex" + "path/filepath" + + "github.com/kata-containers/runtime/virtcontainers/device/api" + "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/utils" +) + +const maxDevIDSize = 31 + +// Drive represents a block storage drive which may be used in case the storage +// driver has an underlying block storage device. +type Drive struct { + + // Path to the disk-image/device which will be used with this drive + File string + + // Format of the drive + Format string + + // ID is used to identify this drive in the hypervisor options. + ID string + + // Index assigned to the drive. In case of virtio-scsi, this is used as SCSI LUN index + Index int + + // PCIAddr is the PCI address used to identify the slot at which the drive is attached. + PCIAddr string +} + +// BlockDevice refers to a block storage device implementation. +type BlockDevice struct { + DevType config.DeviceType + DeviceInfo config.DeviceInfo + + // SCSI Address of the block device, in case the device is attached using SCSI driver + // SCSI address is in the format SCSI-Id:LUN + SCSIAddr string + + // Path at which the device appears inside the VM, outside of the container mount namespace + VirtPath string + + // PCI Slot of the block device + PCIAddr string + + BlockDrive *Drive +} + +// NewBlockDevice creates a new block device based on DeviceInfo +func NewBlockDevice(devInfo config.DeviceInfo) *BlockDevice { + return &BlockDevice{ + DevType: config.DeviceBlock, + DeviceInfo: devInfo, + } +} + +// Attach is standard interface of api.Device, it's used to add device to some +// DeviceReceiver +func (device *BlockDevice) Attach(devReceiver api.DeviceReceiver) (err error) { + randBytes, err := utils.GenerateRandomBytes(8) + if err != nil { + return err + } + + device.DeviceInfo.ID = hex.EncodeToString(randBytes) + + // Increment the block index for the sandbox. This is used to determine the name + // for the block device in the case where the block device is used as container + // rootfs and the predicted block device name needs to be provided to the agent. + index, err := devReceiver.GetAndSetSandboxBlockIndex() + + defer func() { + if err != nil { + devReceiver.DecrementSandboxBlockIndex() + } + }() + + if err != nil { + return err + } + + drive := Drive{ + File: device.DeviceInfo.HostPath, + Format: "raw", + ID: utils.MakeNameID("drive", device.DeviceInfo.ID, maxDevIDSize), + Index: index, + } + + deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Attaching block device") + device.BlockDrive = &drive + if err = devReceiver.HotplugAddDevice(device, config.DeviceBlock); err != nil { + return err + } + + device.DeviceInfo.Hotplugged = true + + driveName, err := utils.GetVirtDriveName(index) + if err != nil { + return err + } + + customOptions := device.DeviceInfo.DriverOptions + if customOptions != nil && customOptions["block-driver"] == "virtio-blk" { + device.VirtPath = filepath.Join("/dev", driveName) + device.PCIAddr = drive.PCIAddr + } else { + scsiAddr, err := utils.GetSCSIAddress(index) + if err != nil { + return err + } + + device.SCSIAddr = scsiAddr + } + + return nil +} + +// Detach is standard interface of api.Device, it's used to remove device from some +// DeviceReceiver +func (device *BlockDevice) Detach(devReceiver api.DeviceReceiver) error { + if device.DeviceInfo.Hotplugged { + deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Unplugging block device") + + if err := devReceiver.HotplugRemoveDevice(device, config.DeviceBlock); err != nil { + deviceLogger().WithError(err).Error("Failed to unplug block device") + return err + } + + } + return nil +} + +// DeviceType is standard interface of api.Device, it returns device type +func (device *BlockDevice) DeviceType() config.DeviceType { + return device.DevType +} diff --git a/virtcontainers/device/drivers/generic.go b/virtcontainers/device/drivers/generic.go new file mode 100644 index 0000000000..9609d811f5 --- /dev/null +++ b/virtcontainers/device/drivers/generic.go @@ -0,0 +1,41 @@ +// Copyright (c) 2017-2018 Intel Corporation +// Copyright (c) 2018 Huawei Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package drivers + +import ( + "github.com/kata-containers/runtime/virtcontainers/device/api" + "github.com/kata-containers/runtime/virtcontainers/device/config" +) + +// GenericDevice refers to a device that is neither a VFIO device or block device. +type GenericDevice struct { + DevType config.DeviceType + DeviceInfo config.DeviceInfo +} + +// NewGenericDevice creates a new GenericDevice +func NewGenericDevice(devInfo config.DeviceInfo) *GenericDevice { + return &GenericDevice{ + DevType: config.DeviceGeneric, + DeviceInfo: devInfo, + } +} + +// Attach is standard interface of api.Device +func (device *GenericDevice) Attach(devReceiver api.DeviceReceiver) error { + return nil +} + +// Detach is standard interface of api.Device +func (device *GenericDevice) Detach(devReceiver api.DeviceReceiver) error { + return nil +} + +// DeviceType is standard interface of api.Device, it returns device type +func (device *GenericDevice) DeviceType() config.DeviceType { + return device.DevType +} diff --git a/virtcontainers/device/drivers/utils.go b/virtcontainers/device/drivers/utils.go new file mode 100644 index 0000000000..c60eef0f07 --- /dev/null +++ b/virtcontainers/device/drivers/utils.go @@ -0,0 +1,17 @@ +// Copyright (c) 2017-2018 Intel Corporation +// Copyright (c) 2018 Huawei Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package drivers + +import ( + "github.com/sirupsen/logrus" + + "github.com/kata-containers/runtime/virtcontainers/device/api" +) + +func deviceLogger() *logrus.Entry { + return api.DeviceLogger() +} diff --git a/virtcontainers/device/drivers/vfio.go b/virtcontainers/device/drivers/vfio.go new file mode 100644 index 0000000000..d8a68dbb92 --- /dev/null +++ b/virtcontainers/device/drivers/vfio.go @@ -0,0 +1,178 @@ +// Copyright (c) 2017-2018 Intel Corporation +// Copyright (c) 2018 Huawei Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package drivers + +import ( + "encoding/hex" + "fmt" + "io/ioutil" + "path/filepath" + "strings" + + "github.com/sirupsen/logrus" + + "github.com/kata-containers/runtime/virtcontainers/device/api" + "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/utils" +) + +// bind/unbind paths to aid in SRIOV VF bring-up/restore +var ( + pciDriverUnbindPath = "/sys/bus/pci/devices/%s/driver/unbind" + pciDriverBindPath = "/sys/bus/pci/drivers/%s/bind" + vfioNewIDPath = "/sys/bus/pci/drivers/vfio-pci/new_id" + vfioRemoveIDPath = "/sys/bus/pci/drivers/vfio-pci/remove_id" +) + +// VFIODevice is a vfio device meant to be passed to the hypervisor +// to be used by the Virtual Machine. +type VFIODevice struct { + DevType config.DeviceType + DeviceInfo config.DeviceInfo + BDF string +} + +// NewVFIODevice create a new VFIO device +func NewVFIODevice(devInfo config.DeviceInfo) *VFIODevice { + return &VFIODevice{ + DevType: config.DeviceVFIO, + DeviceInfo: devInfo, + } +} + +// Attach is standard interface of api.Device, it's used to add device to some +// DeviceReceiver +func (device *VFIODevice) Attach(devReceiver api.DeviceReceiver) error { + vfioGroup := filepath.Base(device.DeviceInfo.HostPath) + iommuDevicesPath := filepath.Join(config.SysIOMMUPath, vfioGroup, "devices") + + deviceFiles, err := ioutil.ReadDir(iommuDevicesPath) + if err != nil { + return err + } + + // Pass all devices in iommu group + for _, deviceFile := range deviceFiles { + + //Get bdf of device eg 0000:00:1c.0 + deviceBDF, err := getBDF(deviceFile.Name()) + if err != nil { + return err + } + + device.BDF = deviceBDF + + randBytes, err := utils.GenerateRandomBytes(8) + if err != nil { + return err + } + device.DeviceInfo.ID = hex.EncodeToString(randBytes) + + if err := devReceiver.HotplugAddDevice(device, config.DeviceVFIO); err != nil { + deviceLogger().WithError(err).Error("Failed to add device") + return err + } + + deviceLogger().WithFields(logrus.Fields{ + "device-group": device.DeviceInfo.HostPath, + "device-type": "vfio-passthrough", + }).Info("Device group attached") + } + + return nil +} + +// Detach is standard interface of api.Device, it's used to remove device from some +// DeviceReceiver +func (device *VFIODevice) Detach(devReceiver api.DeviceReceiver) error { + return nil +} + +// DeviceType is standard interface of api.Device, it returns device type +func (device *VFIODevice) DeviceType() config.DeviceType { + return device.DevType +} + +// getBDF returns the BDF of pci device +// Expected input strng format is []:[][].[] eg. 0000:02:10.0 +func getBDF(deviceSysStr string) (string, error) { + tokens := strings.Split(deviceSysStr, ":") + + if len(tokens) != 3 { + return "", fmt.Errorf("Incorrect number of tokens found while parsing bdf for device : %s", deviceSysStr) + } + + tokens = strings.SplitN(deviceSysStr, ":", 2) + return tokens[1], nil +} + +// BindDevicetoVFIO binds the device to vfio driver after unbinding from host. +// Will be called by a network interface or a generic pcie device. +func BindDevicetoVFIO(bdf, hostDriver, vendorDeviceID string) error { + + // Unbind from the host driver + unbindDriverPath := fmt.Sprintf(pciDriverUnbindPath, bdf) + deviceLogger().WithFields(logrus.Fields{ + "device-bdf": bdf, + "driver-path": unbindDriverPath, + }).Info("Unbinding device from driver") + + if err := utils.WriteToFile(unbindDriverPath, []byte(bdf)); err != nil { + return err + } + + // Add device id to vfio driver. + deviceLogger().WithFields(logrus.Fields{ + "vendor-device-id": vendorDeviceID, + "vfio-new-id-path": vfioNewIDPath, + }).Info("Writing vendor-device-id to vfio new-id path") + + if err := utils.WriteToFile(vfioNewIDPath, []byte(vendorDeviceID)); err != nil { + return err + } + + // Bind to vfio-pci driver. + bindDriverPath := fmt.Sprintf(pciDriverBindPath, "vfio-pci") + + api.DeviceLogger().WithFields(logrus.Fields{ + "device-bdf": bdf, + "driver-path": bindDriverPath, + }).Info("Binding device to vfio driver") + + // Device may be already bound at this time because of earlier write to new_id, ignore error + utils.WriteToFile(bindDriverPath, []byte(bdf)) + + return nil +} + +// BindDevicetoHost binds the device to the host driver driver after unbinding from vfio-pci. +func BindDevicetoHost(bdf, hostDriver, vendorDeviceID string) error { + // Unbind from vfio-pci driver + unbindDriverPath := fmt.Sprintf(pciDriverUnbindPath, bdf) + api.DeviceLogger().WithFields(logrus.Fields{ + "device-bdf": bdf, + "driver-path": unbindDriverPath, + }).Info("Unbinding device from driver") + + if err := utils.WriteToFile(unbindDriverPath, []byte(bdf)); err != nil { + return err + } + + // To prevent new VFs from binding to VFIO-PCI, remove_id + if err := utils.WriteToFile(vfioRemoveIDPath, []byte(vendorDeviceID)); err != nil { + return err + } + + // Bind back to host driver + bindDriverPath := fmt.Sprintf(pciDriverBindPath, hostDriver) + api.DeviceLogger().WithFields(logrus.Fields{ + "device-bdf": bdf, + "driver-path": bindDriverPath, + }).Info("Binding back device to host driver") + + return utils.WriteToFile(bindDriverPath, []byte(bdf)) +} diff --git a/virtcontainers/device/drivers/vfio_test.go b/virtcontainers/device/drivers/vfio_test.go new file mode 100644 index 0000000000..cf60098adf --- /dev/null +++ b/virtcontainers/device/drivers/vfio_test.go @@ -0,0 +1,37 @@ +// Copyright (c) 2017-2018 Intel Corporation +// Copyright (c) 2018 Huawei Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package drivers + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetBDF(t *testing.T) { + type testData struct { + deviceStr string + expectedBDF string + } + + data := []testData{ + {"0000:02:10.0", "02:10.0"}, + {"0000:0210.0", ""}, + {"test", ""}, + {"", ""}, + } + + for _, d := range data { + deviceBDF, err := getBDF(d.deviceStr) + assert.Equal(t, d.expectedBDF, deviceBDF) + if d.expectedBDF == "" { + assert.NotNil(t, err) + } else { + assert.Nil(t, err) + } + } +} diff --git a/virtcontainers/device/drivers/vhost_user.go b/virtcontainers/device/drivers/vhost_user.go new file mode 100644 index 0000000000..0bd13d1db9 --- /dev/null +++ b/virtcontainers/device/drivers/vhost_user.go @@ -0,0 +1,29 @@ +// Copyright (c) 2017-2018 Intel Corporation +// Copyright (c) 2018 Huawei Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package drivers + +import ( + "encoding/hex" + + "github.com/kata-containers/runtime/virtcontainers/device/api" + "github.com/kata-containers/runtime/virtcontainers/utils" +) + +// vhostUserAttach handles the common logic among all of the vhost-user device's +// attach functions +func vhostUserAttach(device api.VhostUserDevice, devReceiver api.DeviceReceiver) error { + // generate a unique ID to be used for hypervisor commandline fields + randBytes, err := utils.GenerateRandomBytes(8) + if err != nil { + return err + } + id := hex.EncodeToString(randBytes) + + device.Attrs().ID = id + + return devReceiver.AddVhostUserDevice(device, device.Type()) +} diff --git a/virtcontainers/device/drivers/vhost_user_blk.go b/virtcontainers/device/drivers/vhost_user_blk.go new file mode 100644 index 0000000000..b03fedd161 --- /dev/null +++ b/virtcontainers/device/drivers/vhost_user_blk.go @@ -0,0 +1,48 @@ +// Copyright (c) 2017-2018 Intel Corporation +// Copyright (c) 2018 Huawei Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package drivers + +import ( + "github.com/kata-containers/runtime/virtcontainers/device/api" + "github.com/kata-containers/runtime/virtcontainers/device/config" +) + +// VhostUserBlkDevice is a block vhost-user based device +type VhostUserBlkDevice struct { + config.VhostUserDeviceAttrs +} + +// Attrs returns the VhostUserDeviceAttrs associated with the vhost-user device +func (vhostUserBlkDevice *VhostUserBlkDevice) Attrs() *config.VhostUserDeviceAttrs { + return &vhostUserBlkDevice.VhostUserDeviceAttrs +} + +// Type returns the type associated with the vhost-user device +func (vhostUserBlkDevice *VhostUserBlkDevice) Type() config.DeviceType { + return config.VhostUserBlk +} + +// +// VhostUserBlkDevice's implementation of the device interface: +// + +// Attach is standard interface of api.Device, it's used to add device to some +// DeviceReceiver +func (vhostUserBlkDevice *VhostUserBlkDevice) Attach(devReceiver api.DeviceReceiver) (err error) { + return vhostUserAttach(vhostUserBlkDevice, devReceiver) +} + +// Detach is standard interface of api.Device, it's used to remove device from some +// DeviceReceiver +func (vhostUserBlkDevice *VhostUserBlkDevice) Detach(devReceiver api.DeviceReceiver) error { + return nil +} + +// DeviceType is standard interface of api.Device, it returns device type +func (vhostUserBlkDevice *VhostUserBlkDevice) DeviceType() config.DeviceType { + return vhostUserBlkDevice.DevType +} diff --git a/virtcontainers/device/drivers/vhost_user_net.go b/virtcontainers/device/drivers/vhost_user_net.go new file mode 100644 index 0000000000..7bf7517b9c --- /dev/null +++ b/virtcontainers/device/drivers/vhost_user_net.go @@ -0,0 +1,49 @@ +// Copyright (c) 2017-2018 Intel Corporation +// Copyright (c) 2018 Huawei Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package drivers + +import ( + "github.com/kata-containers/runtime/virtcontainers/device/api" + "github.com/kata-containers/runtime/virtcontainers/device/config" +) + +// VhostUserNetDevice is a network vhost-user based device +type VhostUserNetDevice struct { + config.VhostUserDeviceAttrs + MacAddress string +} + +// Attrs returns the VhostUserDeviceAttrs associated with the vhost-user device +func (vhostUserNetDevice *VhostUserNetDevice) Attrs() *config.VhostUserDeviceAttrs { + return &vhostUserNetDevice.VhostUserDeviceAttrs +} + +// Type returns the type associated with the vhost-user device +func (vhostUserNetDevice *VhostUserNetDevice) Type() config.DeviceType { + return config.VhostUserNet +} + +// +// VhostUserNetDevice's implementation of the device interface: +// + +// Attach is standard interface of api.Device, it's used to add device to some +// DeviceReceiver +func (vhostUserNetDevice *VhostUserNetDevice) Attach(devReceiver api.DeviceReceiver) (err error) { + return vhostUserAttach(vhostUserNetDevice, devReceiver) +} + +// Detach is standard interface of api.Device, it's used to remove device from some +// DeviceReceiver +func (vhostUserNetDevice *VhostUserNetDevice) Detach(devReceiver api.DeviceReceiver) error { + return nil +} + +// DeviceType is standard interface of api.Device, it returns device type +func (vhostUserNetDevice *VhostUserNetDevice) DeviceType() config.DeviceType { + return vhostUserNetDevice.DevType +} diff --git a/virtcontainers/device/drivers/vhost_user_scsi.go b/virtcontainers/device/drivers/vhost_user_scsi.go new file mode 100644 index 0000000000..6d485baec8 --- /dev/null +++ b/virtcontainers/device/drivers/vhost_user_scsi.go @@ -0,0 +1,48 @@ +// Copyright (c) 2017-2018 Intel Corporation +// Copyright (c) 2018 Huawei Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package drivers + +import ( + "github.com/kata-containers/runtime/virtcontainers/device/api" + "github.com/kata-containers/runtime/virtcontainers/device/config" +) + +// VhostUserSCSIDevice is a SCSI vhost-user based device +type VhostUserSCSIDevice struct { + config.VhostUserDeviceAttrs +} + +// Attrs returns the VhostUserDeviceAttrs associated with the vhost-user device +func (vhostUserSCSIDevice *VhostUserSCSIDevice) Attrs() *config.VhostUserDeviceAttrs { + return &vhostUserSCSIDevice.VhostUserDeviceAttrs +} + +// Type returns the type associated with the vhost-user device +func (vhostUserSCSIDevice *VhostUserSCSIDevice) Type() config.DeviceType { + return config.VhostUserSCSI +} + +// +// VhostUserSCSIDevice's implementation of the device interface: +// + +// Attach is standard interface of api.Device, it's used to add device to some +// DeviceReceiver +func (vhostUserSCSIDevice *VhostUserSCSIDevice) Attach(devReceiver api.DeviceReceiver) (err error) { + return vhostUserAttach(vhostUserSCSIDevice, devReceiver) +} + +// Detach is standard interface of api.Device, it's used to remove device from some +// DeviceReceiver +func (vhostUserSCSIDevice *VhostUserSCSIDevice) Detach(devReceiver api.DeviceReceiver) error { + return nil +} + +// DeviceType is standard interface of api.Device, it returns device type +func (vhostUserSCSIDevice *VhostUserSCSIDevice) DeviceType() config.DeviceType { + return vhostUserSCSIDevice.DevType +} diff --git a/virtcontainers/device/manager/manager.go b/virtcontainers/device/manager/manager.go new file mode 100644 index 0000000000..0e30e6e5b2 --- /dev/null +++ b/virtcontainers/device/manager/manager.go @@ -0,0 +1,79 @@ +// Copyright (c) 2017-2018 Intel Corporation +// Copyright (c) 2018 Huawei Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package manager + +import ( + "github.com/sirupsen/logrus" + + "github.com/kata-containers/runtime/virtcontainers/device/api" + "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/device/drivers" +) + +const ( + // VirtioBlock indicates block driver is virtio-blk based + VirtioBlock string = "virtio-blk" + // VirtioSCSI indicates block driver is virtio-scsi based + VirtioSCSI string = "virtio-scsi" +) + +type deviceManager struct { + blockDriver string +} + +func deviceLogger() *logrus.Entry { + return api.DeviceLogger().WithField("subsystem", "device") +} + +// createDevice creates one device based on DeviceInfo +func (dm *deviceManager) createDevice(devInfo config.DeviceInfo) (api.Device, error) { + path, err := config.GetHostPathFunc(devInfo) + if err != nil { + return nil, err + } + + devInfo.HostPath = path + if isVFIO(path) { + return drivers.NewVFIODevice(devInfo), nil + } else if isBlock(devInfo) { + if devInfo.DriverOptions == nil { + devInfo.DriverOptions = make(map[string]string) + } + devInfo.DriverOptions["block-driver"] = dm.blockDriver + return drivers.NewBlockDevice(devInfo), nil + } else { + deviceLogger().WithField("device", path).Info("Device has not been passed to the container") + return drivers.NewGenericDevice(devInfo), nil + } +} + +// NewDevices creates bundles of devices based on array of DeviceInfo +func (dm *deviceManager) NewDevices(devInfos []config.DeviceInfo) ([]api.Device, error) { + var devices []api.Device + + for _, devInfo := range devInfos { + device, err := dm.createDevice(devInfo) + if err != nil { + return nil, err + } + devices = append(devices, device) + } + + return devices, nil +} + +// NewDeviceManager creates a deviceManager object behaved as api.DeviceManager +func NewDeviceManager(blockDriver string) api.DeviceManager { + dm := &deviceManager{} + if blockDriver == VirtioBlock { + dm.blockDriver = VirtioBlock + } else { + dm.blockDriver = VirtioSCSI + } + + return dm +} diff --git a/virtcontainers/device/manager/manager_test.go b/virtcontainers/device/manager/manager_test.go new file mode 100644 index 0000000000..3b2fc57688 --- /dev/null +++ b/virtcontainers/device/manager/manager_test.go @@ -0,0 +1,200 @@ +// Copyright (c) 2017 Intel Corporation +// Copyright (c) 2018 Huawei Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package manager + +import ( + "io/ioutil" + "os" + "path/filepath" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/kata-containers/runtime/virtcontainers/device/api" + "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/device/drivers" +) + +const fileMode0640 = os.FileMode(0640) + +// dirMode is the permission bits used for creating a directory +const dirMode = os.FileMode(0750) | os.ModeDir + +func TestNewDevices(t *testing.T) { + dm := &deviceManager{ + blockDriver: VirtioBlock, + } + savedSysDevPrefix := config.SysDevPrefix + + major := int64(252) + minor := int64(3) + + tmpDir, err := ioutil.TempDir("", "") + assert.Nil(t, err) + + config.SysDevPrefix = tmpDir + defer func() { + os.RemoveAll(tmpDir) + config.SysDevPrefix = savedSysDevPrefix + }() + + path := "/dev/vfio/2" + deviceInfo := config.DeviceInfo{ + ContainerPath: "", + Major: major, + Minor: minor, + UID: 2, + GID: 2, + DevType: "c", + } + + _, err = dm.NewDevices([]config.DeviceInfo{deviceInfo}) + assert.NotNil(t, err) + + format := strconv.FormatInt(major, 10) + ":" + strconv.FormatInt(minor, 10) + ueventPathPrefix := filepath.Join(config.SysDevPrefix, "char", format) + ueventPath := filepath.Join(ueventPathPrefix, "uevent") + + // Return true for non-existent /sys/dev path. + deviceInfo.ContainerPath = path + _, err = dm.NewDevices([]config.DeviceInfo{deviceInfo}) + assert.Nil(t, err) + + err = os.MkdirAll(ueventPathPrefix, dirMode) + assert.Nil(t, err) + + // Should return error for bad data in uevent file + content := []byte("nonkeyvaluedata") + err = ioutil.WriteFile(ueventPath, content, fileMode0640) + assert.Nil(t, err) + + _, err = dm.NewDevices([]config.DeviceInfo{deviceInfo}) + assert.NotNil(t, err) + + content = []byte("MAJOR=252\nMINOR=3\nDEVNAME=vfio/2") + err = ioutil.WriteFile(ueventPath, content, fileMode0640) + assert.Nil(t, err) + + devices, err := dm.NewDevices([]config.DeviceInfo{deviceInfo}) + assert.Nil(t, err) + + assert.Equal(t, len(devices), 1) + vfioDev, ok := devices[0].(*drivers.VFIODevice) + assert.True(t, ok) + assert.Equal(t, vfioDev.DeviceInfo.HostPath, path) + assert.Equal(t, vfioDev.DeviceInfo.ContainerPath, path) + assert.Equal(t, vfioDev.DeviceInfo.DevType, "c") + assert.Equal(t, vfioDev.DeviceInfo.Major, major) + assert.Equal(t, vfioDev.DeviceInfo.Minor, minor) + assert.Equal(t, vfioDev.DeviceInfo.UID, uint32(2)) + assert.Equal(t, vfioDev.DeviceInfo.GID, uint32(2)) +} + +func TestAttachVFIODevice(t *testing.T) { + dm := &deviceManager{ + blockDriver: VirtioBlock, + } + tmpDir, err := ioutil.TempDir("", "") + assert.Nil(t, err) + os.RemoveAll(tmpDir) + + testFDIOGroup := "2" + testDeviceBDFPath := "0000:00:1c.0" + + devicesDir := filepath.Join(tmpDir, testFDIOGroup, "devices") + err = os.MkdirAll(devicesDir, dirMode) + assert.Nil(t, err) + + deviceFile := filepath.Join(devicesDir, testDeviceBDFPath) + _, err = os.Create(deviceFile) + assert.Nil(t, err) + + savedIOMMUPath := config.SysIOMMUPath + config.SysIOMMUPath = tmpDir + + defer func() { + config.SysIOMMUPath = savedIOMMUPath + }() + + path := filepath.Join(vfioPath, testFDIOGroup) + deviceInfo := config.DeviceInfo{ + HostPath: path, + ContainerPath: path, + DevType: "c", + } + + device, err := dm.createDevice(deviceInfo) + assert.Nil(t, err) + _, ok := device.(*drivers.VFIODevice) + assert.True(t, ok) + + devReceiver := &api.MockDeviceReceiver{} + err = device.Attach(devReceiver) + assert.Nil(t, err) + + err = device.Detach(devReceiver) + assert.Nil(t, err) +} + +func TestAttachGenericDevice(t *testing.T) { + dm := &deviceManager{ + blockDriver: VirtioBlock, + } + path := "/dev/tty2" + deviceInfo := config.DeviceInfo{ + HostPath: path, + ContainerPath: path, + DevType: "c", + } + + device, err := dm.createDevice(deviceInfo) + assert.Nil(t, err) + _, ok := device.(*drivers.GenericDevice) + assert.True(t, ok) + + devReceiver := &api.MockDeviceReceiver{} + err = device.Attach(devReceiver) + assert.Nil(t, err) + + err = device.Detach(devReceiver) + assert.Nil(t, err) +} + +func TestAttachBlockDevice(t *testing.T) { + dm := &deviceManager{ + blockDriver: VirtioBlock, + } + path := "/dev/hda" + deviceInfo := config.DeviceInfo{ + HostPath: path, + ContainerPath: path, + DevType: "b", + } + + devReceiver := &api.MockDeviceReceiver{} + device, err := dm.createDevice(deviceInfo) + assert.Nil(t, err) + _, ok := device.(*drivers.BlockDevice) + assert.True(t, ok) + + err = device.Attach(devReceiver) + assert.Nil(t, err) + + err = device.Detach(devReceiver) + assert.Nil(t, err) + + // test virtio SCSI driver + dm.blockDriver = VirtioSCSI + device, err = dm.createDevice(deviceInfo) + assert.Nil(t, err) + err = device.Attach(devReceiver) + assert.Nil(t, err) + + err = device.Detach(devReceiver) + assert.Nil(t, err) +} diff --git a/virtcontainers/device/manager/utils.go b/virtcontainers/device/manager/utils.go new file mode 100644 index 0000000000..0a63537600 --- /dev/null +++ b/virtcontainers/device/manager/utils.go @@ -0,0 +1,41 @@ +// Copyright (c) 2017-2018 Intel Corporation +// Copyright (c) 2018 Huawei Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package manager + +import ( + "path/filepath" + "strings" + + "github.com/kata-containers/runtime/virtcontainers/device/config" +) + +const ( + vfioPath = "/dev/vfio/" +) + +// isVFIO checks if the device provided is a vfio group. +func isVFIO(hostPath string) bool { + // Ignore /dev/vfio/vfio character device + if strings.HasPrefix(hostPath, filepath.Join(vfioPath, "vfio")) { + return false + } + + if strings.HasPrefix(hostPath, vfioPath) && len(hostPath) > len(vfioPath) { + return true + } + + return false +} + +// isBlock checks if the device is a block device. +func isBlock(devInfo config.DeviceInfo) bool { + if devInfo.DevType == "b" { + return true + } + + return false +} diff --git a/virtcontainers/device/manager/utils_test.go b/virtcontainers/device/manager/utils_test.go new file mode 100644 index 0000000000..984338ee4a --- /dev/null +++ b/virtcontainers/device/manager/utils_test.go @@ -0,0 +1,55 @@ +// Copyright (c) 2017 Intel Corporation +// Copyright (c) 2018 Huawei Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package manager + +import ( + "testing" + + "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/stretchr/testify/assert" +) + +func TestIsVFIO(t *testing.T) { + type testData struct { + path string + expected bool + } + + data := []testData{ + {"/dev/vfio/16", true}, + {"/dev/vfio/1", true}, + {"/dev/vfio/", false}, + {"/dev/vfio", false}, + {"/dev/vf", false}, + {"/dev", false}, + {"/dev/vfio/vfio", false}, + {"/dev/vfio/vfio/12", false}, + } + + for _, d := range data { + isVFIO := isVFIO(d.path) + assert.Equal(t, d.expected, isVFIO) + } +} + +func TestIsBlock(t *testing.T) { + type testData struct { + devType string + expected bool + } + + data := []testData{ + {"b", true}, + {"c", false}, + {"u", false}, + } + + for _, d := range data { + isBlock := isBlock(config.DeviceInfo{DevType: d.devType}) + assert.Equal(t, d.expected, isBlock) + } +} diff --git a/virtcontainers/device_test.go b/virtcontainers/device_test.go deleted file mode 100644 index 82783e35f9..0000000000 --- a/virtcontainers/device_test.go +++ /dev/null @@ -1,391 +0,0 @@ -// Copyright (c) 2017 Intel Corporation -// -// SPDX-License-Identifier: Apache-2.0 -// - -package virtcontainers - -import ( - "fmt" - "io/ioutil" - "net" - "os" - "path/filepath" - "strconv" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/vishvananda/netlink" -) - -const fileMode0640 = os.FileMode(0640) - -func TestVhostUserSocketPath(t *testing.T) { - - // First test case: search for existing: - addresses := []netlink.Addr{ - { - IPNet: &net.IPNet{ - IP: net.IPv4(192, 168, 0, 2), - Mask: net.IPv4Mask(192, 168, 0, 2), - }, - }, - { - IPNet: &net.IPNet{ - IP: net.IPv4(192, 168, 0, 1), - Mask: net.IPv4Mask(192, 168, 0, 1), - }, - }, - } - - expectedPath := "/tmp/vhostuser_192.168.0.1" - expectedFileName := "vhu.sock" - expectedResult := fmt.Sprintf("%s/%s", expectedPath, expectedFileName) - - err := os.Mkdir(expectedPath, 0777) - if err != nil { - t.Fatal(err) - } - - _, err = os.Create(expectedResult) - if err != nil { - t.Fatal(err) - } - netinfo := NetworkInfo{ - Addrs: addresses, - } - - path, _ := vhostUserSocketPath(netinfo) - - if path != expectedResult { - t.Fatalf("Got %+v\nExpecting %+v", path, expectedResult) - } - - // Second test case: search doesn't include matching vsock: - addressesFalse := []netlink.Addr{ - { - IPNet: &net.IPNet{ - IP: net.IPv4(192, 168, 0, 4), - Mask: net.IPv4Mask(192, 168, 0, 4), - }, - }, - } - netinfoFail := NetworkInfo{ - Addrs: addressesFalse, - } - - path, _ = vhostUserSocketPath(netinfoFail) - if path != "" { - t.Fatalf("Got %+v\nExpecting %+v", path, "") - } - - err = os.Remove(expectedResult) - if err != nil { - t.Fatal(err) - } - - err = os.Remove(expectedPath) - if err != nil { - t.Fatal(err) - } - -} - -func TestIsVFIO(t *testing.T) { - type testData struct { - path string - expected bool - } - - data := []testData{ - {"/dev/vfio/16", true}, - {"/dev/vfio/1", true}, - {"/dev/vfio/", false}, - {"/dev/vfio", false}, - {"/dev/vf", false}, - {"/dev", false}, - {"/dev/vfio/vfio", false}, - {"/dev/vfio/vfio/12", false}, - } - - for _, d := range data { - isVFIO := isVFIO(d.path) - assert.Equal(t, d.expected, isVFIO) - } -} - -func TestIsBlock(t *testing.T) { - type testData struct { - devType string - expected bool - } - - data := []testData{ - {"b", true}, - {"c", false}, - {"u", false}, - } - - for _, d := range data { - isBlock := isBlock(DeviceInfo{DevType: d.devType}) - assert.Equal(t, d.expected, isBlock) - } -} - -func TestCreateDevice(t *testing.T) { - devInfo := DeviceInfo{ - HostPath: "/dev/vfio/8", - DevType: "b", - } - - device := createDevice(devInfo) - _, ok := device.(*VFIODevice) - assert.True(t, ok) - - devInfo.HostPath = "/dev/sda" - device = createDevice(devInfo) - _, ok = device.(*BlockDevice) - assert.True(t, ok) - - devInfo.HostPath = "/dev/tty" - devInfo.DevType = "c" - device = createDevice(devInfo) - _, ok = device.(*GenericDevice) - assert.True(t, ok) -} - -func TestNewDevices(t *testing.T) { - savedSysDevPrefix := sysDevPrefix - - major := int64(252) - minor := int64(3) - - tmpDir, err := ioutil.TempDir("", "") - assert.Nil(t, err) - - sysDevPrefix = tmpDir - defer func() { - os.RemoveAll(tmpDir) - sysDevPrefix = savedSysDevPrefix - }() - - path := "/dev/vfio/2" - deviceInfo := DeviceInfo{ - ContainerPath: "", - Major: major, - Minor: minor, - UID: 2, - GID: 2, - DevType: "c", - } - - _, err = newDevices([]DeviceInfo{deviceInfo}) - assert.NotNil(t, err) - - format := strconv.FormatInt(major, 10) + ":" + strconv.FormatInt(minor, 10) - ueventPathPrefix := filepath.Join(sysDevPrefix, "char", format) - ueventPath := filepath.Join(ueventPathPrefix, "uevent") - - // Return true for non-existent /sys/dev path. - deviceInfo.ContainerPath = path - _, err = newDevices([]DeviceInfo{deviceInfo}) - assert.Nil(t, err) - - err = os.MkdirAll(ueventPathPrefix, dirMode) - assert.Nil(t, err) - - // Should return error for bad data in uevent file - content := []byte("nonkeyvaluedata") - err = ioutil.WriteFile(ueventPath, content, fileMode0640) - assert.Nil(t, err) - - _, err = newDevices([]DeviceInfo{deviceInfo}) - assert.NotNil(t, err) - - content = []byte("MAJOR=252\nMINOR=3\nDEVNAME=vfio/2") - err = ioutil.WriteFile(ueventPath, content, fileMode0640) - assert.Nil(t, err) - - devices, err := newDevices([]DeviceInfo{deviceInfo}) - assert.Nil(t, err) - - assert.Equal(t, len(devices), 1) - vfioDev, ok := devices[0].(*VFIODevice) - assert.True(t, ok) - assert.Equal(t, vfioDev.DeviceInfo.HostPath, path) - assert.Equal(t, vfioDev.DeviceInfo.ContainerPath, path) - assert.Equal(t, vfioDev.DeviceInfo.DevType, "c") - assert.Equal(t, vfioDev.DeviceInfo.Major, major) - assert.Equal(t, vfioDev.DeviceInfo.Minor, minor) - assert.Equal(t, vfioDev.DeviceInfo.UID, uint32(2)) - assert.Equal(t, vfioDev.DeviceInfo.GID, uint32(2)) -} - -func TestGetBDF(t *testing.T) { - type testData struct { - deviceStr string - expectedBDF string - } - - data := []testData{ - {"0000:02:10.0", "02:10.0"}, - {"0000:0210.0", ""}, - {"test", ""}, - {"", ""}, - } - - for _, d := range data { - deviceBDF, err := getBDF(d.deviceStr) - assert.Equal(t, d.expectedBDF, deviceBDF) - if d.expectedBDF == "" { - assert.NotNil(t, err) - } else { - assert.Nil(t, err) - } - } -} - -func TestAttachVFIODevice(t *testing.T) { - tmpDir, err := ioutil.TempDir("", "") - assert.Nil(t, err) - os.RemoveAll(tmpDir) - - testFDIOGroup := "2" - testDeviceBDFPath := "0000:00:1c.0" - - devicesDir := filepath.Join(tmpDir, testFDIOGroup, "devices") - err = os.MkdirAll(devicesDir, dirMode) - assert.Nil(t, err) - - deviceFile := filepath.Join(devicesDir, testDeviceBDFPath) - _, err = os.Create(deviceFile) - assert.Nil(t, err) - - savedIOMMUPath := sysIOMMUPath - sysIOMMUPath = tmpDir - - defer func() { - sysIOMMUPath = savedIOMMUPath - }() - - path := filepath.Join(vfioPath, testFDIOGroup) - deviceInfo := DeviceInfo{ - HostPath: path, - ContainerPath: path, - DevType: "c", - } - - device := createDevice(deviceInfo) - _, ok := device.(*VFIODevice) - assert.True(t, ok) - - hypervisor := &mockHypervisor{} - err = device.attach(hypervisor, &Container{}) - assert.Nil(t, err) - - err = device.detach(hypervisor) - assert.Nil(t, err) -} - -func TestAttachGenericDevice(t *testing.T) { - path := "/dev/tty2" - deviceInfo := DeviceInfo{ - HostPath: path, - ContainerPath: path, - DevType: "c", - } - - device := createDevice(deviceInfo) - _, ok := device.(*GenericDevice) - assert.True(t, ok) - - hypervisor := &mockHypervisor{} - err := device.attach(hypervisor, &Container{}) - assert.Nil(t, err) - - err = device.detach(hypervisor) - assert.Nil(t, err) -} - -func TestAttachBlockDevice(t *testing.T) { - fs := &filesystem{} - hypervisor := &mockHypervisor{} - - hConfig := HypervisorConfig{ - BlockDeviceDriver: VirtioBlock, - } - - config := &SandboxConfig{ - HypervisorConfig: hConfig, - } - - sandbox := &Sandbox{ - id: testSandboxID, - storage: fs, - hypervisor: hypervisor, - config: config, - } - - contID := "100" - container := Container{ - sandbox: sandbox, - id: contID, - } - - // create state file - path := filepath.Join(runStoragePath, testSandboxID, container.ID()) - err := os.MkdirAll(path, dirMode) - if err != nil { - t.Fatal(err) - } - - defer os.RemoveAll(path) - - stateFilePath := filepath.Join(path, stateFile) - os.Remove(stateFilePath) - - _, err = os.Create(stateFilePath) - if err != nil { - t.Fatal(err) - } - defer os.Remove(stateFilePath) - - path = "/dev/hda" - deviceInfo := DeviceInfo{ - HostPath: path, - ContainerPath: path, - DevType: "b", - } - - device := createDevice(deviceInfo) - _, ok := device.(*BlockDevice) - assert.True(t, ok) - - container.state.State = "" - err = device.attach(hypervisor, &container) - assert.Nil(t, err) - - err = device.detach(hypervisor) - assert.Nil(t, err) - - container.state.State = StateReady - err = device.attach(hypervisor, &container) - assert.Nil(t, err) - - err = device.detach(hypervisor) - assert.Nil(t, err) - - container.sandbox.config.HypervisorConfig.BlockDeviceDriver = VirtioSCSI - err = device.attach(hypervisor, &container) - assert.Nil(t, err) - - err = device.detach(hypervisor) - assert.Nil(t, err) - - container.state.State = StateReady - err = device.attach(hypervisor, &container) - assert.Nil(t, err) - - err = device.detach(hypervisor) - assert.Nil(t, err) -} diff --git a/virtcontainers/filesystem.go b/virtcontainers/filesystem.go index 1af36e4acf..bd92c1c7ce 100644 --- a/virtcontainers/filesystem.go +++ b/virtcontainers/filesystem.go @@ -13,6 +13,10 @@ import ( "path/filepath" "github.com/sirupsen/logrus" + + "github.com/kata-containers/runtime/virtcontainers/device/api" + "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/device/drivers" ) // sandboxResource is an int representing a sandbox resource type. @@ -127,8 +131,8 @@ type resourceStorage interface { storeContainerProcess(sandboxID, containerID string, process Process) error fetchContainerMounts(sandboxID, containerID string) ([]Mount, error) storeContainerMounts(sandboxID, containerID string, mounts []Mount) error - fetchContainerDevices(sandboxID, containerID string) ([]Device, error) - storeContainerDevices(sandboxID, containerID string, devices []Device) error + fetchContainerDevices(sandboxID, containerID string) ([]api.Device, error) + storeContainerDevices(sandboxID, containerID string, devices []api.Device) error } // filesystem is a resourceStorage interface implementation for a local filesystem. @@ -224,7 +228,7 @@ func (fs *filesystem) storeDeviceFile(file string, data interface{}) error { } defer f.Close() - devices, ok := data.([]Device) + devices, ok := data.([]api.Device) if !ok { return fmt.Errorf("Incorrect data type received, Expected []Device") } @@ -233,7 +237,7 @@ func (fs *filesystem) storeDeviceFile(file string, data interface{}) error { for _, d := range devices { tempJSON, _ := json.Marshal(d) typedDevice := TypedDevice{ - Type: d.deviceType(), + Type: string(d.DeviceType()), Data: tempJSON, } typedDevices = append(typedDevices, typedDevice) @@ -263,7 +267,7 @@ func (fs *filesystem) fetchFile(file string, resource sandboxResource, data inte switch resource { case devicesFileType: - devices, ok := data.(*[]Device) + devices, ok := data.(*[]api.Device) if !ok { return fmt.Errorf("Could not cast %v into *[]Device type", data) } @@ -275,36 +279,38 @@ func (fs *filesystem) fetchFile(file string, resource sandboxResource, data inte } // fetchDeviceFile is used for custom unmarshalling of device interface objects. -func (fs *filesystem) fetchDeviceFile(fileData []byte, devices *[]Device) error { +func (fs *filesystem) fetchDeviceFile(fileData []byte, devices *[]api.Device) error { var typedDevices []TypedDevice if err := json.Unmarshal(fileData, &typedDevices); err != nil { return err } - var tempDevices []Device + var tempDevices []api.Device for _, d := range typedDevices { l := fs.Logger().WithField("device-type", d.Type) l.Info("Device type found") switch d.Type { - case DeviceVFIO: - var device VFIODevice + case string(config.DeviceVFIO): + // TODO: remove dependency of drivers package + var device drivers.VFIODevice if err := json.Unmarshal(d.Data, &device); err != nil { return err } tempDevices = append(tempDevices, &device) l.Infof("VFIO device unmarshalled [%v]", device) - case DeviceBlock: - var device BlockDevice + case string(config.DeviceBlock): + // TODO: remove dependency of drivers package + var device drivers.BlockDevice if err := json.Unmarshal(d.Data, &device); err != nil { return err } tempDevices = append(tempDevices, &device) l.Infof("Block Device unmarshalled [%v]", device) - - case DeviceGeneric: - var device GenericDevice + case string(config.DeviceGeneric): + // TODO: remove dependency of drivers package + var device drivers.GenericDevice if err := json.Unmarshal(d.Data, &device); err != nil { return err } @@ -554,7 +560,7 @@ func (fs *filesystem) storeResource(sandboxSpecific bool, sandboxID, containerID case []Mount: return fs.storeMountResource(sandboxSpecific, sandboxID, containerID, resource, file) - case []Device: + case []api.Device: return fs.storeDeviceResource(sandboxSpecific, sandboxID, containerID, resource, file) default: @@ -715,11 +721,11 @@ func (fs *filesystem) fetchContainerMounts(sandboxID, containerID string) ([]Mou return mounts, nil } -func (fs *filesystem) fetchContainerDevices(sandboxID, containerID string) ([]Device, error) { - var devices []Device +func (fs *filesystem) fetchContainerDevices(sandboxID, containerID string) ([]api.Device, error) { + var devices []api.Device if err := fs.fetchResource(false, sandboxID, containerID, devicesFileType, &devices); err != nil { - return []Device{}, err + return []api.Device{}, err } return devices, nil @@ -729,7 +735,7 @@ func (fs *filesystem) storeContainerMounts(sandboxID, containerID string, mounts return fs.storeContainerResource(sandboxID, containerID, mountsFileType, mounts) } -func (fs *filesystem) storeContainerDevices(sandboxID, containerID string, devices []Device) error { +func (fs *filesystem) storeContainerDevices(sandboxID, containerID string, devices []api.Device) error { return fs.storeContainerResource(sandboxID, containerID, devicesFileType, devices) } diff --git a/virtcontainers/filesystem_test.go b/virtcontainers/filesystem_test.go index daea39dcad..7341b61ca5 100644 --- a/virtcontainers/filesystem_test.go +++ b/virtcontainers/filesystem_test.go @@ -12,6 +12,8 @@ import ( "path/filepath" "reflect" "testing" + + "github.com/kata-containers/runtime/virtcontainers/device/manager" ) func TestFilesystemCreateAllResourcesSuccessful(t *testing.T) { @@ -28,9 +30,10 @@ func TestFilesystemCreateAllResourcesSuccessful(t *testing.T) { } sandbox := &Sandbox{ - id: testSandboxID, - storage: fs, - config: sandboxConfig, + id: testSandboxID, + storage: fs, + config: sandboxConfig, + devManager: manager.NewDeviceManager(manager.VirtioBlock), } if err := sandbox.newContainers(); err != nil { diff --git a/virtcontainers/hyperstart_agent.go b/virtcontainers/hyperstart_agent.go index dc22b05927..8e8e635dab 100644 --- a/virtcontainers/hyperstart_agent.go +++ b/virtcontainers/hyperstart_agent.go @@ -15,8 +15,11 @@ import ( "time" proxyClient "github.com/clearcontainers/proxy/client" + "github.com/kata-containers/runtime/virtcontainers/device/drivers" "github.com/kata-containers/runtime/virtcontainers/pkg/hyperstart" ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter" + "github.com/kata-containers/runtime/virtcontainers/utils" + "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" ) @@ -442,13 +445,13 @@ func (h *hyper) startOneContainer(sandbox *Sandbox, c *Container) error { if c.state.Fstype != "" { // Pass a drive name only in case of block driver if sandbox.config.HypervisorConfig.BlockDeviceDriver == VirtioBlock { - driveName, err := getVirtDriveName(c.state.BlockIndex) + driveName, err := utils.GetVirtDriveName(c.state.BlockIndex) if err != nil { return err } container.Image = driveName } else { - scsiAddr, err := getSCSIAddress(c.state.BlockIndex) + scsiAddr, err := utils.GetSCSIAddress(c.state.BlockIndex) if err != nil { return err } @@ -479,7 +482,7 @@ func (h *hyper) startOneContainer(sandbox *Sandbox, c *Container) error { // Append container mounts for block devices passed with --device. for _, device := range c.devices { - d, ok := device.(*BlockDevice) + d, ok := device.(*drivers.BlockDevice) if ok { fsmapDesc := &hyperstart.FsmapDescriptor{ diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go index 277ff06e7c..3dd3a44a66 100644 --- a/virtcontainers/hypervisor.go +++ b/virtcontainers/hypervisor.go @@ -116,16 +116,6 @@ func newHypervisor(hType HypervisorType) (hypervisor, error) { } } -//Generic function for creating a named-id for passing on the hypervisor commandline -func makeNameID(namedType string, id string) string { - nameID := fmt.Sprintf("%s-%s", namedType, id) - if len(nameID) > maxDevIDSize { - nameID = nameID[:maxDevIDSize] - } - - return nameID -} - // Param is a key/value representation for hypervisor and kernel parameters. type Param struct { Key string diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index b76956ed6b..155b4130bc 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -16,16 +16,19 @@ import ( "syscall" "time" - "github.com/gogo/protobuf/proto" kataclient "github.com/kata-containers/agent/protocols/client" "github.com/kata-containers/agent/protocols/grpc" + "github.com/kata-containers/runtime/virtcontainers/device/api" + "github.com/kata-containers/runtime/virtcontainers/device/drivers" vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter" "github.com/kata-containers/runtime/virtcontainers/pkg/uuid" - "golang.org/x/net/context" + "github.com/kata-containers/runtime/virtcontainers/utils" + "github.com/gogo/protobuf/proto" "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" + "golang.org/x/net/context" golangGrpc "google.golang.org/grpc" ) @@ -615,9 +618,9 @@ func constraintGRPCSpec(grpcSpec *grpc.Spec) { } } -func (k *kataAgent) appendDevices(deviceList []*grpc.Device, devices []Device) []*grpc.Device { +func (k *kataAgent) appendDevices(deviceList []*grpc.Device, devices []api.Device) []*grpc.Device { for _, device := range devices { - d, ok := device.(*BlockDevice) + d, ok := device.(*drivers.BlockDevice) if !ok { continue } @@ -694,7 +697,7 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process, rootfs.Driver = kataBlkDevType rootfs.Source = c.state.RootfsPCIAddr } else { - scsiAddr, err := getSCSIAddress(c.state.BlockIndex) + scsiAddr, err := utils.GetSCSIAddress(c.state.BlockIndex) if err != nil { return nil, err } diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go index c2579221e8..2ddb33b8c9 100644 --- a/virtcontainers/kata_agent_test.go +++ b/virtcontainers/kata_agent_test.go @@ -14,13 +14,17 @@ import ( "testing" gpb "github.com/gogo/protobuf/types" - pb "github.com/kata-containers/agent/protocols/grpc" - "github.com/kata-containers/runtime/virtcontainers/pkg/mock" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/stretchr/testify/assert" "github.com/vishvananda/netlink" "golang.org/x/net/context" "google.golang.org/grpc" + + pb "github.com/kata-containers/agent/protocols/grpc" + "github.com/kata-containers/runtime/virtcontainers/device/api" + "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/device/drivers" + "github.com/kata-containers/runtime/virtcontainers/pkg/mock" ) var ( @@ -352,7 +356,7 @@ func TestAppendDevicesEmptyContainerDeviceList(t *testing.T) { devList := []*pb.Device{} expected := []*pb.Device{} - ctrDevices := []Device{} + ctrDevices := []api.Device{} updatedDevList := k.appendDevices(devList, ctrDevices) assert.True(t, reflect.DeepEqual(updatedDevList, expected), @@ -371,9 +375,9 @@ func TestAppendDevices(t *testing.T) { Id: testPCIAddr, }, } - ctrDevices := []Device{ - &BlockDevice{ - DeviceInfo: DeviceInfo{ + ctrDevices := []api.Device{ + &drivers.BlockDevice{ + DeviceInfo: config.DeviceInfo{ ContainerPath: testBlockDeviceCtrPath, }, PCIAddr: testPCIAddr, diff --git a/virtcontainers/mount.go b/virtcontainers/mount.go index 1139a29668..e6d5d3cba9 100644 --- a/virtcontainers/mount.go +++ b/virtcontainers/mount.go @@ -14,6 +14,8 @@ import ( "path/filepath" "strings" "syscall" + + "github.com/kata-containers/runtime/virtcontainers/device/drivers" ) var rootfsDir = "rootfs" @@ -218,39 +220,6 @@ func isDeviceMapper(major, minor int) (bool, error) { return false, err } -// getVirtBlockDriveName returns the disk name format for virtio-blk -// Reference: https://github.com/torvalds/linux/blob/master/drivers/block/virtio_blk.c @c0aa3e0916d7e531e69b02e426f7162dfb1c6c0 -func getVirtDriveName(index int) (string, error) { - if index < 0 { - return "", fmt.Errorf("Index cannot be negative for drive") - } - - // Prefix used for virtio-block devices - const prefix = "vd" - - //Refer to DISK_NAME_LEN: https://github.com/torvalds/linux/blob/08c521a2011ff492490aa9ed6cc574be4235ce2b/include/linux/genhd.h#L61 - diskNameLen := 32 - base := 26 - - suffLen := diskNameLen - len(prefix) - diskLetters := make([]byte, suffLen) - - var i int - - for i = 0; i < suffLen && index >= 0; i++ { - letter := byte('a' + (index % base)) - diskLetters[i] = letter - index = index/base - 1 - } - - if index >= 0 { - return "", fmt.Errorf("Index not supported") - } - - diskName := prefix + reverseString(string(diskLetters[:i])) - return diskName, nil -} - const mountPerm = os.FileMode(0755) // bindMount bind mounts a source in to a destination. This will @@ -314,7 +283,7 @@ type Mount struct { // BlockDevice represents block device that is attached to the // VM in case this mount is a block device file or a directory // backed by a block device. - BlockDevice *BlockDevice + BlockDevice *drivers.BlockDevice } func bindUnmountContainerRootfs(sharedDir, sandboxID, cID string) error { @@ -334,30 +303,3 @@ func bindUnmountAllRootfs(sharedDir string, sandbox *Sandbox) { } } } - -const maxSCSIDevices = 65535 - -// getSCSIIdLun gets the SCSI id and lun, based on the index of the drive being inserted. -// qemu code suggests that scsi-id can take values from 0 to 255 inclusive, while lun can -// take values from 0 to 16383 inclusive. But lun values over 255 do not seem to follow -// consistent SCSI addressing. Hence we limit to 255. -func getSCSIIdLun(index int) (int, int, error) { - if index < 0 { - return -1, -1, fmt.Errorf("Index cannot be negative") - } - - if index > maxSCSIDevices { - return -1, -1, fmt.Errorf("Index cannot be greater than %d, maximum of %d devices are supported", maxSCSIDevices, maxSCSIDevices) - } - - return index / 256, index % 256, nil -} - -func getSCSIAddress(index int) (string, error) { - scsiID, lun, err := getSCSIIdLun(index) - if err != nil { - return "", err - } - - return fmt.Sprintf("%d:%d", scsiID, lun), nil -} diff --git a/virtcontainers/mount_test.go b/virtcontainers/mount_test.go index 3c3b061eac..f12ca71088 100644 --- a/virtcontainers/mount_test.go +++ b/virtcontainers/mount_test.go @@ -15,8 +15,6 @@ import ( "strings" "syscall" "testing" - - "github.com/stretchr/testify/assert" ) func TestIsSystemMount(t *testing.T) { @@ -283,84 +281,3 @@ func TestIsDeviceMapper(t *testing.T) { t.Fatal() } } - -func TestGetVirtDriveNameInvalidIndex(t *testing.T) { - _, err := getVirtDriveName(-1) - - if err == nil { - t.Fatal(err) - } -} - -func TestGetVirtDriveName(t *testing.T) { - tests := []struct { - index int - expectedDrive string - }{ - {0, "vda"}, - {25, "vdz"}, - {27, "vdab"}, - {704, "vdaac"}, - {18277, "vdzzz"}, - } - - for _, test := range tests { - driveName, err := getVirtDriveName(test.index) - if err != nil { - t.Fatal(err) - } - if driveName != test.expectedDrive { - t.Fatalf("Incorrect drive Name: Got: %s, Expecting :%s", driveName, test.expectedDrive) - - } - } -} - -func TestGetSCSIIdLun(t *testing.T) { - tests := []struct { - index int - expectedScsiID int - expectedLun int - }{ - {0, 0, 0}, - {1, 0, 1}, - {2, 0, 2}, - {255, 0, 255}, - {256, 1, 0}, - {257, 1, 1}, - {258, 1, 2}, - {512, 2, 0}, - {513, 2, 1}, - } - - for _, test := range tests { - scsiID, lun, err := getSCSIIdLun(test.index) - assert.Nil(t, err) - - if scsiID != test.expectedScsiID && lun != test.expectedLun { - t.Fatalf("Expecting scsi-id:lun %d:%d, Got %d:%d", test.expectedScsiID, test.expectedLun, scsiID, lun) - } - } - - _, _, err := getSCSIIdLun(maxSCSIDevices + 1) - assert.NotNil(t, err) -} - -func TestGetSCSIAddress(t *testing.T) { - tests := []struct { - index int - expectedSCSIAddress string - }{ - {0, "0:0"}, - {200, "0:200"}, - {255, "0:255"}, - {258, "1:2"}, - {512, "2:0"}, - } - - for _, test := range tests { - scsiAddr, err := getSCSIAddress(test.index) - assert.Nil(t, err) - assert.Equal(t, scsiAddr, test.expectedSCSIAddress) - } -} diff --git a/virtcontainers/network.go b/virtcontainers/network.go index 4d59bdbe59..fb6f109aab 100644 --- a/virtcontainers/network.go +++ b/virtcontainers/network.go @@ -19,12 +19,15 @@ import ( "time" "github.com/containernetworking/plugins/pkg/ns" - "github.com/kata-containers/runtime/virtcontainers/pkg/uuid" "github.com/safchain/ethtool" "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" "github.com/vishvananda/netns" "golang.org/x/sys/unix" + + "github.com/kata-containers/runtime/virtcontainers/device/drivers" + "github.com/kata-containers/runtime/virtcontainers/pkg/uuid" + "github.com/kata-containers/runtime/virtcontainers/utils" ) // NetInterworkingModel defines the network model connecting @@ -261,14 +264,14 @@ func (endpoint *VhostUserEndpoint) SetProperties(properties NetworkInfo) { func (endpoint *VhostUserEndpoint) Attach(h hypervisor) error { networkLogger().Info("Attaching vhostuser based endpoint") - // generate a unique ID to be used for hypervisor commandline fields - randBytes, err := generateRandomBytes(8) + // Generate a unique ID to be used for hypervisor commandline fields + randBytes, err := utils.GenerateRandomBytes(8) if err != nil { return err } id := hex.EncodeToString(randBytes) - d := VhostUserNetDevice{ + d := drivers.VhostUserNetDevice{ MacAddress: endpoint.HardAddr, } d.SocketPath = endpoint.SocketPath @@ -331,7 +334,7 @@ func (endpoint *PhysicalEndpoint) Attach(h hypervisor) error { return err } - d := VFIODevice{ + d := drivers.VFIODevice{ BDF: endpoint.BDF, } @@ -736,7 +739,7 @@ func createFds(device string, numFds int) ([]*os.File, error) { for i := 0; i < numFds; i++ { f, err := os.OpenFile(device, os.O_RDWR, defaultFilePerms) if err != nil { - cleanupFds(fds, i) + utils.CleanupFds(fds, i) return nil, err } fds[i] = f @@ -1347,11 +1350,49 @@ func createPhysicalEndpoint(netInfo NetworkInfo) (*PhysicalEndpoint, error) { } func bindNICToVFIO(endpoint *PhysicalEndpoint) error { - return bindDevicetoVFIO(endpoint.BDF, endpoint.Driver, endpoint.VendorDeviceID) + return drivers.BindDevicetoVFIO(endpoint.BDF, endpoint.Driver, endpoint.VendorDeviceID) } func bindNICToHost(endpoint *PhysicalEndpoint) error { - return bindDevicetoHost(endpoint.BDF, endpoint.Driver, endpoint.VendorDeviceID) + return drivers.BindDevicetoHost(endpoint.BDF, endpoint.Driver, endpoint.VendorDeviceID) +} + +// Long term, this should be made more configurable. For now matching path +// provided by CNM VPP and OVS-DPDK plugins, available at github.com/clearcontainers/vpp and +// github.com/clearcontainers/ovsdpdk. The plugins create the socket on the host system +// using this path. +const hostSocketSearchPath = "/tmp/vhostuser_%s/vhu.sock" + +// findVhostUserNetSocketPath checks if an interface is a dummy placeholder +// for a vhost-user socket, and if it is it returns the path to the socket +func findVhostUserNetSocketPath(netInfo NetworkInfo) (string, error) { + if netInfo.Iface.Name == "lo" { + return "", nil + } + + // check for socket file existence at known location. + for _, addr := range netInfo.Addrs { + socketPath := fmt.Sprintf(hostSocketSearchPath, addr.IPNet.IP) + if _, err := os.Stat(socketPath); err == nil { + return socketPath, nil + } + } + + return "", nil +} + +// vhostUserSocketPath returns the path of the socket discovered. This discovery +// will vary depending on the type of vhost-user socket. +// Today only VhostUserNetDevice is supported. +func vhostUserSocketPath(info interface{}) (string, error) { + + switch v := info.(type) { + case NetworkInfo: + return findVhostUserNetSocketPath(v) + default: + return "", nil + } + } // network is the virtcontainers network interface. diff --git a/virtcontainers/network_test.go b/virtcontainers/network_test.go index b204b862a9..addd5e759e 100644 --- a/virtcontainers/network_test.go +++ b/virtcontainers/network_test.go @@ -6,6 +6,7 @@ package virtcontainers import ( + "fmt" "net" "os" "reflect" @@ -401,3 +402,74 @@ func TestNetInterworkingModelSetModel(t *testing.T) { }) } } + +func TestVhostUserSocketPath(t *testing.T) { + + // First test case: search for existing: + addresses := []netlink.Addr{ + { + IPNet: &net.IPNet{ + IP: net.IPv4(192, 168, 0, 2), + Mask: net.IPv4Mask(192, 168, 0, 2), + }, + }, + { + IPNet: &net.IPNet{ + IP: net.IPv4(192, 168, 0, 1), + Mask: net.IPv4Mask(192, 168, 0, 1), + }, + }, + } + + expectedPath := "/tmp/vhostuser_192.168.0.1" + expectedFileName := "vhu.sock" + expectedResult := fmt.Sprintf("%s/%s", expectedPath, expectedFileName) + + err := os.Mkdir(expectedPath, 0777) + if err != nil { + t.Fatal(err) + } + + _, err = os.Create(expectedResult) + if err != nil { + t.Fatal(err) + } + netinfo := NetworkInfo{ + Addrs: addresses, + } + + path, _ := vhostUserSocketPath(netinfo) + + if path != expectedResult { + t.Fatalf("Got %+v\nExpecting %+v", path, expectedResult) + } + + // Second test case: search doesn't include matching vsock: + addressesFalse := []netlink.Addr{ + { + IPNet: &net.IPNet{ + IP: net.IPv4(192, 168, 0, 4), + Mask: net.IPv4Mask(192, 168, 0, 4), + }, + }, + } + netinfoFail := NetworkInfo{ + Addrs: addressesFalse, + } + + path, _ = vhostUserSocketPath(netinfoFail) + if path != "" { + t.Fatalf("Got %+v\nExpecting %+v", path, "") + } + + err = os.Remove(expectedResult) + if err != nil { + t.Fatal(err) + } + + err = os.Remove(expectedPath) + if err != nil { + t.Fatal(err) + } + +} diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go index c39c4a9ff6..d302d98fac 100644 --- a/virtcontainers/pkg/oci/utils.go +++ b/virtcontainers/pkg/oci/utils.go @@ -15,12 +15,14 @@ import ( "strings" criContainerdAnnotations "github.com/containerd/cri-containerd/pkg/annotations" - vc "github.com/kata-containers/runtime/virtcontainers" - vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" - dockershimAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations/dockershim" crioAnnotations "github.com/kubernetes-incubator/cri-o/pkg/annotations" spec "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" + + vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/device/config" + vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" + dockershimAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations/dockershim" ) type annotationContainerType struct { @@ -207,7 +209,7 @@ func contains(s []string, e string) bool { return false } -func newLinuxDeviceInfo(d spec.LinuxDevice) (*vc.DeviceInfo, error) { +func newLinuxDeviceInfo(d spec.LinuxDevice) (*config.DeviceInfo, error) { allowedDeviceTypes := []string{"c", "b", "u", "p"} if !contains(allowedDeviceTypes, d.Type) { @@ -218,7 +220,7 @@ func newLinuxDeviceInfo(d spec.LinuxDevice) (*vc.DeviceInfo, error) { return nil, fmt.Errorf("Path cannot be empty for device") } - deviceInfo := vc.DeviceInfo{ + deviceInfo := config.DeviceInfo{ ContainerPath: d.Path, DevType: d.Type, Major: d.Major, @@ -239,18 +241,18 @@ func newLinuxDeviceInfo(d spec.LinuxDevice) (*vc.DeviceInfo, error) { return &deviceInfo, nil } -func containerDeviceInfos(spec CompatOCISpec) ([]vc.DeviceInfo, error) { +func containerDeviceInfos(spec CompatOCISpec) ([]config.DeviceInfo, error) { ociLinuxDevices := spec.Spec.Linux.Devices if ociLinuxDevices == nil { - return []vc.DeviceInfo{}, nil + return []config.DeviceInfo{}, nil } - var devices []vc.DeviceInfo + var devices []config.DeviceInfo for _, d := range ociLinuxDevices { linuxDeviceInfo, err := newLinuxDeviceInfo(d) if err != nil { - return []vc.DeviceInfo{}, err + return []config.DeviceInfo{}, err } devices = append(devices, *linuxDeviceInfo) diff --git a/virtcontainers/pkg/oci/utils_test.go b/virtcontainers/pkg/oci/utils_test.go index 80f051847c..70f22f3e39 100644 --- a/virtcontainers/pkg/oci/utils_test.go +++ b/virtcontainers/pkg/oci/utils_test.go @@ -15,11 +15,13 @@ import ( "reflect" "testing" - vc "github.com/kata-containers/runtime/virtcontainers" - vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" "github.com/kubernetes-incubator/cri-o/pkg/annotations" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/stretchr/testify/assert" + + vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/device/config" + vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" ) const tempBundlePath = "/tmp/virtc/ocibundle/" @@ -46,15 +48,15 @@ func TestMinimalSandboxConfig(t *testing.T) { t.Fatal(err) } - savedFunc := vc.GetHostPathFunc + savedFunc := config.GetHostPathFunc // Simply assign container path to host path for device. - vc.GetHostPathFunc = func(devInfo vc.DeviceInfo) (string, error) { + config.GetHostPathFunc = func(devInfo config.DeviceInfo) (string, error) { return devInfo.ContainerPath, nil } defer func() { - vc.GetHostPathFunc = savedFunc + config.GetHostPathFunc = savedFunc }() runtimeConfig := RuntimeConfig{ @@ -130,7 +132,7 @@ func TestMinimalSandboxConfig(t *testing.T) { t.Fatal(err) } - devInfo := vc.DeviceInfo{ + devInfo := config.DeviceInfo{ ContainerPath: "/dev/vfio/17", Major: 242, Minor: 0, @@ -139,7 +141,7 @@ func TestMinimalSandboxConfig(t *testing.T) { GID: 0, } - expectedDeviceInfo := []vc.DeviceInfo{ + expectedDeviceInfo := []config.DeviceInfo{ devInfo, } diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index fa1680d640..dc0e25112d 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -16,6 +16,10 @@ import ( govmmQemu "github.com/intel/govmm/qemu" "github.com/kata-containers/runtime/virtcontainers/pkg/uuid" "github.com/sirupsen/logrus" + + "github.com/kata-containers/runtime/virtcontainers/device/api" + deviceDrivers "github.com/kata-containers/runtime/virtcontainers/device/drivers" + "github.com/kata-containers/runtime/virtcontainers/utils" ) type qmpChannel struct { @@ -593,7 +597,7 @@ func (q *qemu) removeDeviceFromBridge(ID string) error { return err } -func (q *qemu) hotplugBlockDevice(drive *Drive, op operation) error { +func (q *qemu) hotplugBlockDevice(drive *deviceDrivers.Drive, op operation) error { defer func(qemu *qemu) { if q.qmpMonitorCh.qmp != nil { q.qmpMonitorCh.qmp.Shutdown() @@ -634,7 +638,7 @@ func (q *qemu) hotplugBlockDevice(drive *Drive, op operation) error { bus := scsiControllerID + ".0" // Get SCSI-id and LUN based on the order of attaching drives. - scsiID, lun, err := getSCSIIdLun(drive.Index) + scsiID, lun, err := utils.GetSCSIIdLun(drive.Index) if err != nil { return err } @@ -662,7 +666,7 @@ func (q *qemu) hotplugBlockDevice(drive *Drive, op operation) error { return nil } -func (q *qemu) hotplugVFIODevice(device VFIODevice, op operation) error { +func (q *qemu) hotplugVFIODevice(device deviceDrivers.VFIODevice, op operation) error { defer func(qemu *qemu) { if q.qmpMonitorCh.qmp != nil { q.qmpMonitorCh.qmp.Shutdown() @@ -703,13 +707,15 @@ func (q *qemu) hotplugVFIODevice(device VFIODevice, op operation) error { func (q *qemu) hotplugDevice(devInfo interface{}, devType deviceType, op operation) error { switch devType { case blockDev: - drive := devInfo.(*Drive) + // TODO: find a way to remove dependency of deviceDrivers lib @weizhang555 + drive := devInfo.(*deviceDrivers.Drive) return q.hotplugBlockDevice(drive, op) case cpuDev: vcpus := devInfo.(uint32) return q.hotplugCPUs(vcpus, op) case vfioDev: - device := devInfo.(VFIODevice) + // TODO: find a way to remove dependency of deviceDrivers lib @weizhang555 + device := devInfo.(deviceDrivers.VFIODevice) return q.hotplugVFIODevice(device, op) default: return fmt.Errorf("cannot hotplug device: unsupported device type '%v'", devType) @@ -840,6 +846,13 @@ func (q *qemu) resumeSandbox() error { // addDevice will add extra devices to Qemu command line. func (q *qemu) addDevice(devInfo interface{}, devType deviceType) error { + switch devType { + case vhostuserDev: + vhostDev := devInfo.(api.VhostUserDevice) + q.qemuConfig.Devices = q.arch.appendVhostUserDevice(q.qemuConfig.Devices, vhostDev) + return nil + } + switch v := devInfo.(type) { case Volume: q.qemuConfig.Devices = q.arch.append9PVolume(q.qemuConfig.Devices, v) @@ -847,17 +860,10 @@ func (q *qemu) addDevice(devInfo interface{}, devType deviceType) error { q.qemuConfig.Devices = q.arch.appendSocket(q.qemuConfig.Devices, v) case Endpoint: q.qemuConfig.Devices = q.arch.appendNetwork(q.qemuConfig.Devices, v) - case Drive: + case deviceDrivers.Drive: q.qemuConfig.Devices = q.arch.appendBlockDevice(q.qemuConfig.Devices, v) - //vhostUserDevice is an interface, hence the pointer for Net, SCSI and Blk: - case VhostUserNetDevice: - q.qemuConfig.Devices = q.arch.appendVhostUserDevice(q.qemuConfig.Devices, &v) - case VhostUserSCSIDevice: - q.qemuConfig.Devices = q.arch.appendVhostUserDevice(q.qemuConfig.Devices, &v) - case VhostUserBlkDevice: - q.qemuConfig.Devices = q.arch.appendVhostUserDevice(q.qemuConfig.Devices, &v) - case VFIODevice: + case deviceDrivers.VFIODevice: q.qemuConfig.Devices = q.arch.appendVFIODevice(q.qemuConfig.Devices, v) default: break diff --git a/virtcontainers/qemu_arch_base.go b/virtcontainers/qemu_arch_base.go index 18336ae474..28a06d2a00 100644 --- a/virtcontainers/qemu_arch_base.go +++ b/virtcontainers/qemu_arch_base.go @@ -12,6 +12,10 @@ import ( "strconv" govmmQemu "github.com/intel/govmm/qemu" + + "github.com/kata-containers/runtime/virtcontainers/device/api" + "github.com/kata-containers/runtime/virtcontainers/device/drivers" + "github.com/kata-containers/runtime/virtcontainers/utils" ) type qemuArch interface { @@ -71,13 +75,13 @@ type qemuArch interface { appendNetwork(devices []govmmQemu.Device, endpoint Endpoint) []govmmQemu.Device // appendBlockDevice appends a block drive to devices - appendBlockDevice(devices []govmmQemu.Device, drive Drive) []govmmQemu.Device + appendBlockDevice(devices []govmmQemu.Device, drive drivers.Drive) []govmmQemu.Device // appendVhostUserDevice appends a vhost user device to devices - appendVhostUserDevice(devices []govmmQemu.Device, vhostUserDevice VhostUserDevice) []govmmQemu.Device + appendVhostUserDevice(devices []govmmQemu.Device, vhostUserDevice api.VhostUserDevice) []govmmQemu.Device // appendVFIODevice appends a VFIO device to devices - appendVFIODevice(devices []govmmQemu.Device, vfioDevice VFIODevice) []govmmQemu.Device + appendVFIODevice(devices []govmmQemu.Device, vfioDevice drivers.VFIODevice) []govmmQemu.Device } type qemuArchBase struct { @@ -281,14 +285,14 @@ func (q *qemuArchBase) appendImage(devices []govmmQemu.Device, path string) ([]g return nil, err } - randBytes, err := generateRandomBytes(8) + randBytes, err := utils.GenerateRandomBytes(8) if err != nil { return nil, err } - id := makeNameID("image", hex.EncodeToString(randBytes)) + id := utils.MakeNameID("image", hex.EncodeToString(randBytes), maxDevIDSize) - drive := Drive{ + drive := drivers.Drive{ File: path, Format: "raw", ID: id, @@ -306,7 +310,7 @@ func (q *qemuArchBase) appendSCSIController(devices []govmmQemu.Device, enableIO var t *govmmQemu.IOThread if enableIOThreads { - randBytes, _ := generateRandomBytes(8) + randBytes, _ := utils.GenerateRandomBytes(8) t = &govmmQemu.IOThread{ ID: fmt.Sprintf("%s-%s", "iothread", hex.EncodeToString(randBytes)), @@ -432,7 +436,7 @@ func (q *qemuArchBase) appendNetwork(devices []govmmQemu.Device, endpoint Endpoi return devices } -func (q *qemuArchBase) appendBlockDevice(devices []govmmQemu.Device, drive Drive) []govmmQemu.Device { +func (q *qemuArchBase) appendBlockDevice(devices []govmmQemu.Device, drive drivers.Drive) []govmmQemu.Device { if drive.File == "" || drive.ID == "" || drive.Format == "" { return devices } @@ -456,28 +460,29 @@ func (q *qemuArchBase) appendBlockDevice(devices []govmmQemu.Device, drive Drive return devices } -func (q *qemuArchBase) appendVhostUserDevice(devices []govmmQemu.Device, vhostUserDevice VhostUserDevice) []govmmQemu.Device { +func (q *qemuArchBase) appendVhostUserDevice(devices []govmmQemu.Device, vhostUserDevice api.VhostUserDevice) []govmmQemu.Device { qemuVhostUserDevice := govmmQemu.VhostUserDevice{} + // TODO: find a way to remove dependency of drivers package switch vhostUserDevice := vhostUserDevice.(type) { - case *VhostUserNetDevice: - qemuVhostUserDevice.TypeDevID = makeNameID("net", vhostUserDevice.ID) + case *drivers.VhostUserNetDevice: + qemuVhostUserDevice.TypeDevID = utils.MakeNameID("net", vhostUserDevice.ID, maxDevIDSize) qemuVhostUserDevice.Address = vhostUserDevice.MacAddress - case *VhostUserSCSIDevice: - qemuVhostUserDevice.TypeDevID = makeNameID("scsi", vhostUserDevice.ID) - case *VhostUserBlkDevice: + case *drivers.VhostUserSCSIDevice: + qemuVhostUserDevice.TypeDevID = utils.MakeNameID("scsi", vhostUserDevice.ID, maxDevIDSize) + case *drivers.VhostUserBlkDevice: } qemuVhostUserDevice.VhostUserType = govmmQemu.VhostUserDeviceType(vhostUserDevice.Type()) qemuVhostUserDevice.SocketPath = vhostUserDevice.Attrs().SocketPath - qemuVhostUserDevice.CharDevID = makeNameID("char", vhostUserDevice.Attrs().ID) + qemuVhostUserDevice.CharDevID = utils.MakeNameID("char", vhostUserDevice.Attrs().ID, maxDevIDSize) devices = append(devices, qemuVhostUserDevice) return devices } -func (q *qemuArchBase) appendVFIODevice(devices []govmmQemu.Device, vfioDevice VFIODevice) []govmmQemu.Device { +func (q *qemuArchBase) appendVFIODevice(devices []govmmQemu.Device, vfioDevice drivers.VFIODevice) []govmmQemu.Device { if vfioDevice.BDF == "" { return devices } diff --git a/virtcontainers/qemu_arch_base_test.go b/virtcontainers/qemu_arch_base_test.go index aae32aa963..24100d1e84 100644 --- a/virtcontainers/qemu_arch_base_test.go +++ b/virtcontainers/qemu_arch_base_test.go @@ -13,6 +13,9 @@ import ( govmmQemu "github.com/intel/govmm/qemu" "github.com/stretchr/testify/assert" + + "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/device/drivers" ) const ( @@ -205,11 +208,11 @@ func testQemuArchBaseAppend(t *testing.T, structure interface{}, expected []govm devices = qemuArchBase.appendSocket(devices, s) case []Volume: devices = qemuArchBase.append9PVolumes(devices, s) - case Drive: + case drivers.Drive: devices = qemuArchBase.appendBlockDevice(devices, s) - case VFIODevice: + case drivers.VFIODevice: devices = qemuArchBase.appendVFIODevice(devices, s) - case VhostUserNetDevice: + case drivers.VhostUserNetDevice: devices = qemuArchBase.appendVhostUserDevice(devices, &s) } @@ -400,7 +403,7 @@ func TestQemuArchBaseAppendBlockDevice(t *testing.T) { }, } - drive := Drive{ + drive := drivers.Drive{ File: file, Format: format, ID: id, @@ -420,11 +423,11 @@ func TestQemuArchBaseAppendVhostUserDevice(t *testing.T) { CharDevID: fmt.Sprintf("char-%s", id), TypeDevID: fmt.Sprintf("net-%s", id), Address: macAddress, - VhostUserType: VhostUserNet, + VhostUserType: config.VhostUserNet, }, } - vhostUserDevice := VhostUserNetDevice{ + vhostUserDevice := drivers.VhostUserNetDevice{ MacAddress: macAddress, } vhostUserDevice.ID = id @@ -442,7 +445,7 @@ func TestQemuArchBaseAppendVFIODevice(t *testing.T) { }, } - vfDevice := VFIODevice{ + vfDevice := drivers.VFIODevice{ BDF: bdf, } diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index 720e1a1947..89ae785510 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -15,6 +15,11 @@ import ( "syscall" "github.com/sirupsen/logrus" + + "github.com/kata-containers/runtime/virtcontainers/device/api" + "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/device/drivers" + deviceManager "github.com/kata-containers/runtime/virtcontainers/device/manager" ) // controlSocket is the sandbox control socket. @@ -226,26 +231,6 @@ func (s *Sockets) String() string { return strings.Join(sockSlice, " ") } -// Drive represents a block storage drive which may be used in case the storage -// driver has an underlying block storage device. -type Drive struct { - - // Path to the disk-image/device which will be used with this drive - File string - - // Format of the drive - Format string - - // ID is used to identify this drive in the hypervisor options. - ID string - - // Index assigned to the drive. In case of virtio-scsi, this is used as SCSI LUN index - Index int - - // PCIAddr is the PCI address used to identify the slot at which the drive is attached. - PCIAddr string -} - // EnvVar is a key/value structure representing a command // environment variable. type EnvVar struct { @@ -453,6 +438,8 @@ type Sandbox struct { config *SandboxConfig + devManager api.DeviceManager + volumes []Volume containers []*Container @@ -739,6 +726,7 @@ func newSandbox(sandboxConfig SandboxConfig) (*Sandbox, error) { storage: &filesystem{}, network: network, config: &sandboxConfig, + devManager: deviceManager.NewDeviceManager(sandboxConfig.HypervisorConfig.BlockDeviceDriver), volumes: sandboxConfig.Volumes, runPath: filepath.Join(runStoragePath, sandboxConfig.ID), configPath: filepath.Join(configStoragePath, sandboxConfig.ID), @@ -1328,3 +1316,71 @@ func togglePauseSandbox(sandboxID string, pause bool) (*Sandbox, error) { return p, nil } + +// HotplugAddDevice is used for add a device to sandbox +// Sandbox implement DeviceReceiver interface from device/api/interface.go +func (s *Sandbox) HotplugAddDevice(device api.Device, devType config.DeviceType) error { + switch devType { + case config.DeviceVFIO: + vfioDevice, ok := device.(*drivers.VFIODevice) + if !ok { + return fmt.Errorf("device type mismatch, expect device type to be %s", devType) + } + return s.hypervisor.hotplugAddDevice(*vfioDevice, vfioDev) + case config.DeviceBlock: + blockDevice, ok := device.(*drivers.BlockDevice) + if !ok { + return fmt.Errorf("device type mismatch, expect device type to be %s", devType) + } + return s.hypervisor.hotplugAddDevice(blockDevice.BlockDrive, blockDev) + case config.DeviceGeneric: + // TODO: what? + return nil + } + return nil +} + +// HotplugRemoveDevice is used for removing a device from sandbox +// Sandbox implement DeviceReceiver interface from device/api/interface.go +func (s *Sandbox) HotplugRemoveDevice(device api.Device, devType config.DeviceType) error { + switch devType { + case config.DeviceVFIO: + vfioDevice, ok := device.(*drivers.VFIODevice) + if !ok { + return fmt.Errorf("device type mismatch, expect device type to be %s", devType) + } + return s.hypervisor.hotplugRemoveDevice(*vfioDevice, vfioDev) + case config.DeviceBlock: + blockDevice, ok := device.(*drivers.BlockDevice) + if !ok { + return fmt.Errorf("device type mismatch, expect device type to be %s", devType) + } + return s.hypervisor.hotplugRemoveDevice(blockDevice.BlockDrive, blockDev) + case config.DeviceGeneric: + // TODO: what? + return nil + } + return nil +} + +// GetAndSetSandboxBlockIndex is used for getting and setting virtio-block indexes +// Sandbox implement DeviceReceiver interface from device/api/interface.go +func (s *Sandbox) GetAndSetSandboxBlockIndex() (int, error) { + return s.getAndSetSandboxBlockIndex() +} + +// DecrementSandboxBlockIndex decrease block indexes +// Sandbox implement DeviceReceiver interface from device/api/interface.go +func (s *Sandbox) DecrementSandboxBlockIndex() error { + return s.decrementSandboxBlockIndex() +} + +// AddVhostUserDevice adds a vhost user device to sandbox +// Sandbox implement DeviceReceiver interface from device/api/interface.go +func (s *Sandbox) AddVhostUserDevice(devInfo api.VhostUserDevice, devType config.DeviceType) error { + switch devType { + case config.VhostUserSCSI, config.VhostUserNet, config.VhostUserBlk: + return s.hypervisor.addDevice(devInfo, vhostuserDev) + } + return fmt.Errorf("unsupported device type") +} diff --git a/virtcontainers/sandbox_test.go b/virtcontainers/sandbox_test.go index 291df54dda..eec5a0d2aa 100644 --- a/virtcontainers/sandbox_test.go +++ b/virtcontainers/sandbox_test.go @@ -16,8 +16,13 @@ import ( "syscall" "testing" - "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" "github.com/stretchr/testify/assert" + + "github.com/kata-containers/runtime/virtcontainers/device/api" + "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/device/drivers" + "github.com/kata-containers/runtime/virtcontainers/device/manager" + "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" ) func newHypervisorConfig(kernelParams []Param, hParams []Param) HypervisorConfig { @@ -36,7 +41,7 @@ func testCreateSandbox(t *testing.T, id string, nmodel NetworkModel, nconfig NetworkConfig, containers []ContainerConfig, volumes []Volume) (*Sandbox, error) { - config := SandboxConfig{ + sconfig := SandboxConfig{ ID: id, HypervisorType: htype, HypervisorConfig: hconfig, @@ -47,7 +52,7 @@ func testCreateSandbox(t *testing.T, id string, Containers: containers, } - sandbox, err := createSandbox(config) + sandbox, err := createSandbox(sconfig) if err != nil { return nil, fmt.Errorf("Could not create sandbox: %s", err) } @@ -1123,6 +1128,8 @@ func TestContainerStateSetFstype(t *testing.T) { } } +const vfioPath = "/dev/vfio/" + func TestSandboxAttachDevicesVFIO(t *testing.T) { tmpDir, err := ioutil.TempDir("", "") assert.Nil(t, err) @@ -1139,24 +1146,24 @@ func TestSandboxAttachDevicesVFIO(t *testing.T) { _, err = os.Create(deviceFile) assert.Nil(t, err) - savedIOMMUPath := sysIOMMUPath - sysIOMMUPath = tmpDir + savedIOMMUPath := config.SysIOMMUPath + config.SysIOMMUPath = tmpDir defer func() { - sysIOMMUPath = savedIOMMUPath + config.SysIOMMUPath = savedIOMMUPath }() path := filepath.Join(vfioPath, testFDIOGroup) - deviceInfo := DeviceInfo{ + deviceInfo := config.DeviceInfo{ HostPath: path, ContainerPath: path, DevType: "c", } - vfioDevice := newVFIODevice(deviceInfo) + vfioDevice := drivers.NewVFIODevice(deviceInfo) c := &Container{ id: "100", - devices: []Device{ + devices: []api.Device{ vfioDevice, }, } @@ -1529,3 +1536,89 @@ func TestContainerProcessIOStream(t *testing.T) { _, _, _, err = s.IOStream(contID, execID) assert.Nil(t, err, "Winsize process failed: %v", err) } + +func TestAttachBlockDevice(t *testing.T) { + fs := &filesystem{} + hypervisor := &mockHypervisor{} + + hConfig := HypervisorConfig{ + BlockDeviceDriver: VirtioBlock, + } + + sconfig := &SandboxConfig{ + HypervisorConfig: hConfig, + } + + sandbox := &Sandbox{ + id: testSandboxID, + storage: fs, + hypervisor: hypervisor, + config: sconfig, + } + + contID := "100" + container := Container{ + sandbox: sandbox, + id: contID, + } + + // create state file + path := filepath.Join(runStoragePath, testSandboxID, container.ID()) + err := os.MkdirAll(path, dirMode) + if err != nil { + t.Fatal(err) + } + + defer os.RemoveAll(path) + + stateFilePath := filepath.Join(path, stateFile) + os.Remove(stateFilePath) + + _, err = os.Create(stateFilePath) + if err != nil { + t.Fatal(err) + } + defer os.Remove(stateFilePath) + + path = "/dev/hda" + deviceInfo := config.DeviceInfo{ + HostPath: path, + ContainerPath: path, + DevType: "b", + } + + dm := manager.NewDeviceManager(VirtioBlock) + devices, err := dm.NewDevices([]config.DeviceInfo{deviceInfo}) + assert.Nil(t, err) + device := devices[0] + _, ok := device.(*drivers.BlockDevice) + assert.True(t, ok) + + container.state.State = "" + err = device.Attach(sandbox) + assert.Nil(t, err) + + err = device.Detach(sandbox) + assert.Nil(t, err) + + container.state.State = StateReady + err = device.Attach(sandbox) + assert.Nil(t, err) + + err = device.Detach(sandbox) + assert.Nil(t, err) + + container.sandbox.config.HypervisorConfig.BlockDeviceDriver = VirtioSCSI + err = device.Attach(sandbox) + assert.Nil(t, err) + + err = device.Detach(sandbox) + assert.Nil(t, err) + + container.state.State = StateReady + err = device.Attach(sandbox) + assert.Nil(t, err) + + err = device.Detach(sandbox) + assert.Nil(t, err) +} diff --git a/virtcontainers/utils.go b/virtcontainers/utils.go deleted file mode 100644 index 078a97f3aa..0000000000 --- a/virtcontainers/utils.go +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright (c) 2017 Intel Corporation -// -// SPDX-License-Identifier: Apache-2.0 -// - -package virtcontainers - -import ( - "crypto/rand" - "fmt" - "os" - "os/exec" -) - -const cpBinaryName = "cp" - -const fileMode0755 = os.FileMode(0755) - -func fileCopy(srcPath, dstPath string) error { - if srcPath == "" { - return fmt.Errorf("Source path cannot be empty") - } - - if dstPath == "" { - return fmt.Errorf("Destination path cannot be empty") - } - - binPath, err := exec.LookPath(cpBinaryName) - if err != nil { - return err - } - - cmd := exec.Command(binPath, srcPath, dstPath) - - return cmd.Run() -} - -func generateRandomBytes(n int) ([]byte, error) { - b := make([]byte, n) - _, err := rand.Read(b) - - if err != nil { - return nil, err - } - - return b, nil -} - -func reverseString(s string) string { - r := []rune(s) - - length := len(r) - for i, j := 0, length-1; i < length/2; i, j = i+1, j-1 { - r[i], r[j] = r[j], r[i] - } - - return string(r) -} - -func cleanupFds(fds []*os.File, numFds int) { - - maxFds := len(fds) - - if numFds < maxFds { - maxFds = numFds - } - - for i := 0; i < maxFds; i++ { - _ = fds[i].Close() - } -} - -// writeToFile opens a file in write only mode and writes bytes to it -func writeToFile(path string, data []byte) error { - f, err := os.OpenFile(path, os.O_WRONLY, fileMode0755) - if err != nil { - return err - } - - defer f.Close() - - if _, err := f.Write(data); err != nil { - return err - } - - return nil -} - -// ConstraintsToVCPUs converts CPU quota and period to vCPUs -func ConstraintsToVCPUs(quota int64, period uint64) uint { - if quota != 0 && period != 0 { - // Use some math magic to round up to the nearest whole vCPU - // (that is, a partial part of a quota request ends up assigning - // a whole vCPU, for instance, a request of 1.5 'cpu quotas' - // will give 2 vCPUs). - // This also has the side effect that we will always allocate - // at least 1 vCPU. - return uint((uint64(quota) + (period - 1)) / period) - } - - return 0 -} diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go new file mode 100644 index 0000000000..06b392931e --- /dev/null +++ b/virtcontainers/utils/utils.go @@ -0,0 +1,176 @@ +// Copyright (c) 2017 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package utils + +import ( + "crypto/rand" + "fmt" + "os" + "os/exec" +) + +const cpBinaryName = "cp" + +const fileMode0755 = os.FileMode(0755) + +// FileCopy copys files from srcPath to dstPath +func FileCopy(srcPath, dstPath string) error { + if srcPath == "" { + return fmt.Errorf("Source path cannot be empty") + } + + if dstPath == "" { + return fmt.Errorf("Destination path cannot be empty") + } + + binPath, err := exec.LookPath(cpBinaryName) + if err != nil { + return err + } + + cmd := exec.Command(binPath, srcPath, dstPath) + + return cmd.Run() +} + +// GenerateRandomBytes generate n random bytes +func GenerateRandomBytes(n int) ([]byte, error) { + b := make([]byte, n) + _, err := rand.Read(b) + + if err != nil { + return nil, err + } + + return b, nil +} + +// ReverseString reverses whole string +func ReverseString(s string) string { + r := []rune(s) + + length := len(r) + for i, j := 0, length-1; i < length/2; i, j = i+1, j-1 { + r[i], r[j] = r[j], r[i] + } + + return string(r) +} + +// CleanupFds closed bundles of open fds in batch +func CleanupFds(fds []*os.File, numFds int) { + maxFds := len(fds) + + if numFds < maxFds { + maxFds = numFds + } + + for i := 0; i < maxFds; i++ { + _ = fds[i].Close() + } +} + +// WriteToFile opens a file in write only mode and writes bytes to it +func WriteToFile(path string, data []byte) error { + f, err := os.OpenFile(path, os.O_WRONLY, fileMode0755) + if err != nil { + return err + } + + defer f.Close() + + if _, err := f.Write(data); err != nil { + return err + } + + return nil +} + +// ConstraintsToVCPUs converts CPU quota and period to vCPUs +func ConstraintsToVCPUs(quota int64, period uint64) uint { + if quota != 0 && period != 0 { + // Use some math magic to round up to the nearest whole vCPU + // (that is, a partial part of a quota request ends up assigning + // a whole vCPU, for instance, a request of 1.5 'cpu quotas' + // will give 2 vCPUs). + // This also has the side effect that we will always allocate + // at least 1 vCPU. + return uint((uint64(quota) + (period - 1)) / period) + } + + return 0 +} + +// GetVirtDriveName returns the disk name format for virtio-blk +// Reference: https://github.com/torvalds/linux/blob/master/drivers/block/virtio_blk.c @c0aa3e0916d7e531e69b02e426f7162dfb1c6c0 +func GetVirtDriveName(index int) (string, error) { + if index < 0 { + return "", fmt.Errorf("Index cannot be negative for drive") + } + + // Prefix used for virtio-block devices + const prefix = "vd" + + //Refer to DISK_NAME_LEN: https://github.com/torvalds/linux/blob/08c521a2011ff492490aa9ed6cc574be4235ce2b/include/linux/genhd.h#L61 + diskNameLen := 32 + base := 26 + + suffLen := diskNameLen - len(prefix) + diskLetters := make([]byte, suffLen) + + var i int + + for i = 0; i < suffLen && index >= 0; i++ { + letter := byte('a' + (index % base)) + diskLetters[i] = letter + index = index/base - 1 + } + + if index >= 0 { + return "", fmt.Errorf("Index not supported") + } + + diskName := prefix + ReverseString(string(diskLetters[:i])) + return diskName, nil +} + +const maxSCSIDevices = 65535 + +// GetSCSIIdLun gets the SCSI id and lun, based on the index of the drive being inserted. +// qemu code suggests that scsi-id can take values from 0 to 255 inclusive, while lun can +// take values from 0 to 16383 inclusive. But lun values over 255 do not seem to follow +// consistent SCSI addressing. Hence we limit to 255. +func GetSCSIIdLun(index int) (int, int, error) { + if index < 0 { + return -1, -1, fmt.Errorf("Index cannot be negative") + } + + if index > maxSCSIDevices { + return -1, -1, fmt.Errorf("Index cannot be greater than %d, maximum of %d devices are supported", maxSCSIDevices, maxSCSIDevices) + } + + return index / 256, index % 256, nil +} + +// GetSCSIAddress gets scsiID and lun from index, and combined them into a scsi ID +func GetSCSIAddress(index int) (string, error) { + scsiID, lun, err := GetSCSIIdLun(index) + if err != nil { + return "", err + } + + return fmt.Sprintf("%d:%d", scsiID, lun), nil +} + +// MakeNameID is generic function for creating a named-id for passing on the hypervisor commandline +func MakeNameID(namedType, id string, maxLen int) string { + nameID := fmt.Sprintf("%s-%s", namedType, id) + if len(nameID) > maxLen { + nameID = nameID[:maxLen] + } + + return nameID +} diff --git a/virtcontainers/utils_test.go b/virtcontainers/utils/utils_test.go similarity index 61% rename from virtcontainers/utils_test.go rename to virtcontainers/utils/utils_test.go index ece0595b08..5369b70b47 100644 --- a/virtcontainers/utils_test.go +++ b/virtcontainers/utils/utils_test.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package virtcontainers +package utils import ( "io/ioutil" @@ -40,7 +40,7 @@ func TestFileCopySuccessful(t *testing.T) { t.Fatal(err) } - if err := fileCopy(srcFile.Name(), dstPath); err != nil { + if err := FileCopy(srcFile.Name(), dstPath); err != nil { t.Fatal(err) } @@ -77,13 +77,13 @@ func TestFileCopySuccessful(t *testing.T) { } func TestFileCopySourceEmptyFailure(t *testing.T) { - if err := fileCopy("", "testDst"); err == nil { + if err := FileCopy("", "testDst"); err == nil { t.Fatal("This test should fail because source path is empty") } } func TestFileCopyDestinationEmptyFailure(t *testing.T) { - if err := fileCopy("testSrc", ""); err == nil { + if err := FileCopy("testSrc", ""); err == nil { t.Fatal("This test should fail because destination path is empty") } } @@ -104,14 +104,14 @@ func TestFileCopySourceNotExistFailure(t *testing.T) { t.Fatal(err) } - if err := fileCopy(srcPath, "testDest"); err == nil { + if err := FileCopy(srcPath, "testDest"); err == nil { t.Fatal("This test should fail because source file does not exist") } } func TestGenerateRandomBytes(t *testing.T) { bytesNeeded := 8 - randBytes, err := generateRandomBytes(bytesNeeded) + randBytes, err := GenerateRandomBytes(bytesNeeded) if err != nil { t.Fatal(err) } @@ -123,7 +123,7 @@ func TestGenerateRandomBytes(t *testing.T) { func TestRevereString(t *testing.T) { str := "Teststr" - reversed := reverseString(str) + reversed := ReverseString(str) if reversed != "rtstseT" { t.Fatal("Incorrect String Reversal") @@ -131,7 +131,7 @@ func TestRevereString(t *testing.T) { } func TestWriteToFile(t *testing.T) { - err := writeToFile("/file-does-not-exist", []byte("test-data")) + err := WriteToFile("/file-does-not-exist", []byte("test-data")) assert.NotNil(t, err) tmpFile, err := ioutil.TempFile("", "test_append_file") @@ -143,7 +143,7 @@ func TestWriteToFile(t *testing.T) { tmpFile.Close() testData := []byte("test-data") - err = writeToFile(filename, testData) + err = WriteToFile(filename, testData) assert.Nil(t, err) data, err := ioutil.ReadFile(filename) @@ -168,3 +168,84 @@ func TestConstraintsToVCPUs(t *testing.T) { vcpus = ConstraintsToVCPUs(4000, 1200) assert.Equal(expectedVCPUs, vcpus) } + +func TestGetVirtDriveNameInvalidIndex(t *testing.T) { + _, err := GetVirtDriveName(-1) + + if err == nil { + t.Fatal(err) + } +} + +func TestGetVirtDriveName(t *testing.T) { + tests := []struct { + index int + expectedDrive string + }{ + {0, "vda"}, + {25, "vdz"}, + {27, "vdab"}, + {704, "vdaac"}, + {18277, "vdzzz"}, + } + + for _, test := range tests { + driveName, err := GetVirtDriveName(test.index) + if err != nil { + t.Fatal(err) + } + if driveName != test.expectedDrive { + t.Fatalf("Incorrect drive Name: Got: %s, Expecting :%s", driveName, test.expectedDrive) + + } + } +} + +func TestGetSCSIIdLun(t *testing.T) { + tests := []struct { + index int + expectedScsiID int + expectedLun int + }{ + {0, 0, 0}, + {1, 0, 1}, + {2, 0, 2}, + {255, 0, 255}, + {256, 1, 0}, + {257, 1, 1}, + {258, 1, 2}, + {512, 2, 0}, + {513, 2, 1}, + } + + for _, test := range tests { + scsiID, lun, err := GetSCSIIdLun(test.index) + assert.Nil(t, err) + + if scsiID != test.expectedScsiID && lun != test.expectedLun { + t.Fatalf("Expecting scsi-id:lun %d:%d, Got %d:%d", test.expectedScsiID, test.expectedLun, scsiID, lun) + } + } + + _, _, err := GetSCSIIdLun(maxSCSIDevices + 1) + assert.NotNil(t, err) +} + +func TestGetSCSIAddress(t *testing.T) { + tests := []struct { + index int + expectedSCSIAddress string + }{ + {0, "0:0"}, + {200, "0:200"}, + {255, "0:255"}, + {258, "1:2"}, + {512, "2:0"}, + } + + for _, test := range tests { + scsiAddr, err := GetSCSIAddress(test.index) + assert.Nil(t, err) + assert.Equal(t, scsiAddr, test.expectedSCSIAddress) + } +}