Merge pull request #138 from amshinde/block-volumes

volumes: Attach volumes that are block device files as block devices

virtcontainers/container.go

@@ -25,6 +25,7 @@ import (
 	"time"
 
 	"github.com/sirupsen/logrus"
+	"golang.org/x/sys/unix"
 )
 
 // Process gathers data related to a container process.
@@ -311,6 +312,32 @@ func (c *Container) mountSharedDirMounts(hostSharedDir, guestSharedDir string) (
 			continue
 		}
 
+		var stat unix.Stat_t
+		if err := unix.Stat(m.Source, &stat); err != nil {
+			return nil, err
+		}
+
+		// Check if mount is a block device file. If it is, the block device will be attached to the host
+		// instead of passing this as a shared mount.
+		if c.checkBlockDeviceSupport() && stat.Mode&unix.S_IFBLK == unix.S_IFBLK {
+			b := &BlockDevice{
+				DeviceType: DeviceBlock,
+				DeviceInfo: DeviceInfo{
+					HostPath:      m.Source,
+					ContainerPath: m.Destination,
+					DevType:       "b",
+				},
+			}
+
+			// Attach this block device, all other devices passed in the config have been attached at this point
+			if err := b.attach(c.pod.hypervisor, c); err != nil {
+				return nil, err
+			}
+
+			c.mounts[idx].BlockDevice = b
+			continue
+		}
+
 		randBytes, err := generateRandomBytes(8)
 		if err != nil {
 			return nil, err

virtcontainers/hyperstart_agent.go

@@ -413,6 +413,17 @@ func (h *hyper) stopPod(pod Pod) error {
 	return h.proxy.stop(pod, h.state.ProxyPid)
 }
 
+// handleBlockVolumes handles volumes that are block device files, by
+// appending the block device to the list of devices associated with the
+// container.
+func (h *hyper) handleBlockVolumes(c *Container) {
+	for _, m := range c.mounts {
+		if m.BlockDevice != nil {
+			c.devices = append(c.devices, m.BlockDevice)
+		}
+	}
+}
+
 func (h *hyper) startOneContainer(pod Pod, c *Container) error {
 	process, err := h.buildHyperContainerProcess(c.config.Cmd)
 	if err != nil {
@@ -475,6 +486,8 @@ func (h *hyper) startOneContainer(pod Pod, c *Container) error {
 
 	fsmap := fsMapFromMounts(newMounts)
 
+	h.handleBlockVolumes(c)
+
 	// Append container mounts for block devices passed with --device.
 	for _, device := range c.devices {
 		d, ok := device.(*BlockDevice)

virtcontainers/kata_agent.go

@@ -539,6 +539,35 @@ func (k *kataAgent) replaceOCIMountSource(spec *specs.Spec, guestMounts []Mount)
 	return nil
 }
 
+func (k *kataAgent) replaceOCIMountsForStorages(spec *specs.Spec, volumeStorages []*grpc.Storage) error {
+	ociMounts := spec.Mounts
+	var index int
+	var m specs.Mount
+
+	for i, v := range volumeStorages {
+		for index, m = range ociMounts {
+			if m.Destination != v.MountPoint {
+				continue
+			}
+
+			// Create a temporary location to mount the Storage. Mounting to the correct location
+			// will be handled by the OCI mount structure.
+			filename := fmt.Sprintf("%s-%s", uuid.Generate().String(), filepath.Base(m.Destination))
+			path := filepath.Join(kataGuestSharedDir, filename)
+
+			k.Logger().Debugf("Replacing OCI mount source (%s) with %s", m.Source, path)
+			ociMounts[index].Source = path
+			volumeStorages[i].MountPoint = path
+
+			break
+		}
+		if index == len(ociMounts) {
+			return fmt.Errorf("OCI mount not found for block volume %s", v.MountPoint)
+		}
+	}
+	return nil
+}
+
 func constraintGRPCSpec(grpcSpec *grpc.Spec) {
 	// Disable Hooks since they have been handled on the host and there is
 	// no reason to send them to the agent. It would make no sense to try
@@ -720,6 +749,20 @@ func (k *kataAgent) createContainer(pod *Pod, c *Container) (p *Process, err err
 		return nil, err
 	}
 
+	// Append container devices for block devices passed with --device.
+	ctrDevices = k.appendDevices(ctrDevices, c.devices)
+
+	// Handle all the volumes that are block device files.
+	// Note this call modifies the list of container devices to make sure
+	// all hotplugged devices are unplugged, so this needs be done
+	// after devices passed with --device are handled.
+	volumeStorages := k.handleBlockVolumes(c)
+	if err := k.replaceOCIMountsForStorages(ociSpec, volumeStorages); err != nil {
+		return nil, err
+	}
+
+	ctrStorages = append(ctrStorages, volumeStorages...)
+
 	grpcSpec, err := grpc.OCItoGRPC(ociSpec)
 	if err != nil {
 		return nil, err
@@ -732,9 +775,6 @@ func (k *kataAgent) createContainer(pod *Pod, c *Container) (p *Process, err err
 	// irrelevant information to the agent.
 	constraintGRPCSpec(grpcSpec)
 
-	// Append container devices for block devices passed with --device.
-	ctrDevices = k.appendDevices(ctrDevices, c.devices)
-
 	req := &grpc.CreateContainerRequest{
 		ContainerId: c.id,
 		ExecId:      c.id,
@@ -760,6 +800,43 @@ func (k *kataAgent) createContainer(pod *Pod, c *Container) (p *Process, err err
 		k.state.URL, c.config.Cmd, createNSList, enterNSList)
 }
 
+// handleBlockVolumes handles volumes that are block devices files
+// by passing the block devices as Storage to the agent.
+func (k *kataAgent) handleBlockVolumes(c *Container) []*grpc.Storage {
+
+	var volumeStorages []*grpc.Storage
+
+	for _, m := range c.mounts {
+		b := m.BlockDevice
+
+		if b == nil {
+			continue
+		}
+
+		// Add the block device to the list of container devices, to make sure the
+		// device is detached with detachDevices() for a container.
+		c.devices = append(c.devices, b)
+
+		vol := &grpc.Storage{}
+
+		if c.pod.config.HypervisorConfig.BlockDeviceDriver == VirtioBlock {
+			vol.Driver = kataBlkDevType
+			vol.Source = b.VirtPath
+		} else {
+			vol.Driver = kataSCSIDevType
+			vol.Source = b.SCSIAddr
+		}
+
+		vol.MountPoint = b.DeviceInfo.ContainerPath
+		vol.Fstype = "bind"
+		vol.Options = []string{"bind"}
+
+		volumeStorages = append(volumeStorages, vol)
+	}
+
+	return volumeStorages
+}
+
 func (k *kataAgent) startContainer(pod Pod, c *Container) error {
 	req := &grpc.StartContainerRequest{
 		ContainerId: c.id,

virtcontainers/mount.go

@@ -281,6 +281,11 @@ type Mount struct {
 
 	// ReadOnly specifies if the mount should be read only or not
 	ReadOnly bool
+
+	// BlockDevice represents block device that is attached to the
+	// VM in case this mount is a block device file or a directory
+	// backed by a block device.
+	BlockDevice *BlockDevice
 }
 
 func bindUnmountContainerRootfs(sharedDir, podID, cID string) error {