s390x: add virtio-blk-ccw support

Add support for CCW virtio blk. The virtio blk requires a special
handling to the PCI devices.

The device information during the hotplug are updated in '/sys/devices/css0/',
and the disk name can be found under '/bus/ccw/devices'

More detailed info at:https://public.dhe.ibm.com/software/dw/linux390/docu/lhs1vg00.pdf

Fixes: #599

Signed-off-by: Alice Frosi <[email protected]>

device.go

@@ -13,6 +13,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"syscall"
 	"time"
@@ -29,6 +30,7 @@ const (
 	driver9pType        = "9p"
 	driverVirtioFSType  = "virtio-fs"
 	driverBlkType       = "blk"
+	driverBlkCCWType    = "blk-ccw"
 	driverMmioBlkType   = "mmioblk"
 	driverSCSIType      = "scsi"
 	driverNvdimmType    = "nvdimm"
@@ -52,6 +54,15 @@ var (
 	scanSCSIBus         = scanSCSIBusImpl
 )
 
+// CCW variables
+var (
+	channelSubSystem = "/devices/css0"
+	sysCCWBusDir     = sysfsDir + "/bus/ccw/devices"
+	blkCCWSuffix     = "virtio"
+)
+
+const maxDeviceIDValue = 3
+
 // SCSI variables
 var (
 	// Here in "0:0", the first number is the SCSI host number because
@@ -70,6 +81,7 @@ type deviceHandler func(ctx context.Context, device pb.Device, spec *pb.Spec, s
 var deviceHandlerList = map[string]deviceHandler{
 	driverMmioBlkType: virtioMmioBlkDeviceHandler,
 	driverBlkType:     virtioBlkDeviceHandler,
+	driverBlkCCWType:  virtioBlkCCWDeviceHandler,
 	driverSCSIType:    virtioSCSIDeviceHandler,
 	driverNvdimmType:  nvdimmDeviceHandler,
 }
@@ -188,6 +200,21 @@ func virtioMmioBlkDeviceHandler(_ context.Context, device pb.Device, spec *pb.Sp
 	return updateSpecDeviceList(device, spec)
 }
 
+func virtioBlkCCWDeviceHandler(ctx context.Context, device pb.Device, spec *pb.Spec, _ *sandbox) error {
+	devPath, err := getBlkCCWDevPath(ctx, device.Id)
+	if err != nil {
+		return err
+	}
+
+	if devPath == "" {
+		return grpcStatus.Errorf(codes.InvalidArgument,
+			"Storage source is empty")
+	}
+
+	device.VmPath = devPath
+	return updateSpecDeviceList(device, spec)
+}
+
 // device.Id should be the PCI address in the format  "bridgeAddr/deviceAddr".
 // Here, bridgeAddr is the address at which the brige is attached on the root bus,
 // while deviceAddr is the address at which the device is attached on the bridge.
@@ -447,6 +474,92 @@ func getSCSIDevPathImpl(ctx context.Context, scsiAddr string) (string, error) {
 	return filepath.Join(devPrefix, scsiDiskName), nil
 }
 
+// checkCCWBusFormat checks the format for the ccw bus. It needs to be in the form 0.<n>.<dddd>
+// n is the subchannel set ID - integer from 0 up to 3
+// dddd is the device id - integer in hex up to 0xffff
+// See https://www.ibm.com/support/knowledgecenter/en/linuxonibm/com.ibm.linux.z.ldva/ldva_r_XML_Address.html
+func checkCCWBusFormat(bus string) error {
+	busFormat := strings.Split(bus, ".")
+	if len(busFormat) != 3 {
+		return fmt.Errorf("Wrong bus format. It needs to be in the form 0.<n>.<dddd>, got %s", bus)
+	}
+
+	bus0, err := strconv.ParseInt(busFormat[0], 10, 32)
+	if err != nil {
+		return err
+	}
+	if bus0 != 0 {
+		return fmt.Errorf("Wrong bus format. First digit needs to be 0 instead is %d", bus0)
+	}
+
+	bus1, err := strconv.ParseInt(busFormat[1], 10, 32)
+	if err != nil {
+		return err
+	}
+	if bus1 > maxDeviceIDValue {
+		return fmt.Errorf("Wrong bus format. Second digit must be lower than %d instead is %d", maxDeviceIDValue, bus1)
+	}
+
+	if len(busFormat[2]) != 4 {
+		return fmt.Errorf("Wrong bus format. Third digit must be in the form <dddd>, got %s", bus)
+	}
+	busFormat[2] = "0x" + busFormat[2]
+	_, err = strconv.ParseInt(busFormat[2], 0, 32)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// findBlkCCWDevPath finds the CCW block disk name associated with the given CCW block path.
+func findBlkCCWDevPath(blkCCWpath string) (string, error) {
+	files, err := ioutil.ReadDir(blkCCWpath)
+	if err != nil {
+		return "", err
+	}
+
+	for _, f := range files {
+		if strings.Contains(f.Name(), blkCCWSuffix) {
+			subPath := filepath.Join(blkCCWpath, f.Name(), "block")
+			files2, err := ioutil.ReadDir(subPath)
+			if err != nil {
+				return "", err
+			}
+			if len(files2) != 1 {
+				return "", grpcStatus.Errorf(codes.Internal,
+					"Expecting a single blk CCW device in %s found %v",
+					subPath, files2)
+			}
+			return files2[0].Name(), nil
+		}
+	}
+	return "", grpcStatus.Errorf(codes.Internal, "Path %s for blk CCW not found", blkCCWpath)
+}
+
+// getBlkCCWDevPath returns the CCW block path based on the bus ID
+func getBlkCCWDevPath(ctx context.Context, bus string) (string, error) {
+	if err := checkCCWBusFormat(bus); err != nil {
+		return "", err
+	}
+	devPath := filepath.Join(sysCCWBusDir, bus)
+
+	checkUevent := func(uEv *uevent.Uevent) bool {
+		return (uEv.Action == "add" &&
+			strings.Contains(uEv.DevPath, channelSubSystem))
+	}
+	if err := waitForDevice(ctx, devPath, bus, checkUevent); err != nil {
+		return "", err
+	}
+
+	blkCCWDiskName, err := findBlkCCWDevPath(devPath)
+	if err != nil {
+		return "", err
+	}
+
+	return filepath.Join(devPrefix, blkCCWDiskName), nil
+}
+
 func addDevices(ctx context.Context, devices []*pb.Device, spec *pb.Spec, s *sandbox) error {
 	for _, device := range devices {
 		if device == nil {

device_test.go

@@ -779,3 +779,45 @@ func TestGetSCSIDevPath(t *testing.T) {
 	cancel()
 
 }
+
+func TestFindBlkCCWDevPath(t *testing.T) {
+	bus := "0.0.0005"
+	assert := assert.New(t)
+	expectDev := "vdb"
+	dir := fmt.Sprintf("/tmp/sys/bus/ccw/devices/%s/virtio5/block/%s", bus, expectDev)
+	busPath := fmt.Sprintf("/tmp/sys/bus/ccw/devices/%s", bus)
+	err := os.MkdirAll(dir, mountPerm)
+	assert.Nil(err)
+
+	dev, err := findBlkCCWDevPath(busPath)
+	assert.Nil(err)
+
+	if dev != expectDev {
+		t.Errorf("Expected value %s got %s", expectDev, dev)
+	}
+
+	bus = "../dev"
+	busPath = fmt.Sprintf("/tmp/sys/bus/ccw/devices/%s", bus)
+	err = os.MkdirAll(dir, mountPerm)
+	assert.Nil(err)
+	_, err = findBlkCCWDevPath(busPath)
+	assert.NotNil(err, fmt.Sprintf("findBlkCCWDevPath() should have been failed with bus %s", bus))
+
+}
+
+func TestCheckCCWBusFormat(t *testing.T) {
+	assert := assert.New(t)
+
+	wrongBuses := []string{"", "fe.0.0000", "0.5.0000", "some_wrong_path", "0.1.fffff", "0.0.0"}
+	rightBuses := []string{"0.3.abcd", "0.0.0000", "0.1.0000"}
+
+	for _, bus := range wrongBuses {
+		err := checkCCWBusFormat(bus)
+		assert.NotNil(err, fmt.Sprintf("checkCCWBusFormat() should have been failed with bus %s", bus))
+	}
+
+	for _, bus := range rightBuses {
+		err := checkCCWBusFormat(bus)
+		assert.Nil(err)
+	}
+}

mount.go

@@ -213,6 +213,7 @@ var storageHandlerList = map[string]storageHandler{
 	driver9pType:        virtio9pStorageHandler,
 	driverVirtioFSType:  virtioFSStorageHandler,
 	driverBlkType:       virtioBlkStorageHandler,
+	driverBlkCCWType:    virtioBlkCCWStorageHandler,
 	driverMmioBlkType:   virtioMmioBlkStorageHandler,
 	driverSCSIType:      virtioSCSIStorageHandler,
 	driverEphemeralType: ephemeralStorageHandler,
@@ -268,6 +269,20 @@ func virtioMmioBlkStorageHandler(_ context.Context, storage pb.Storage, s *sandb
 	return commonStorageHandler(storage)
 }
 
+// virtioBlkCCWStorageHandler handles the storage for blk ccw driver.
+func virtioBlkCCWStorageHandler(ctx context.Context, storage pb.Storage, s *sandbox) (string, error) {
+	devPath, err := getBlkCCWDevPath(ctx, storage.Source)
+	if err != nil {
+		return "", err
+	}
+	if devPath == "" {
+		return "", grpcStatus.Errorf(codes.InvalidArgument,
+			"Storage source is empty")
+	}
+	storage.Source = devPath
+	return commonStorageHandler(storage)
+}
+
 // virtioFSStorageHandler handles the storage for virtio-fs.
 func virtioFSStorageHandler(_ context.Context, storage pb.Storage, s *sandbox) (string, error) {
 	return commonStorageHandler(storage)