agent: support pmem/nvdimm hotplug

Agent has to wait for pmem/nvdimm devices to appear in the guest.
The agent can use udev and the ACPI device path to identify when a pmem/nvdimm
device has been hotplugged in the guest.
For example when `/dev/pmem1` is hotplugged, the ACPI dev path may look like:

```
/devices/LNXSYSTM:00/LNXSYBUS:00/ACPI0012:00/ndbus0/region1/pfn1.1/block/pmem1
```

The udev routine catches the above path and return the full path to the device:
`/dev/pmem1`

Depends-on: github.com/kata-containers/tests#2356

fixes #753

Signed-off-by: Julio Montes <[email protected]>

agent.go

@@ -740,7 +740,8 @@ func (s *sandbox) listenToUdevEvents() {
 		fieldLogger.Infof("Received add uevent")
 
 		// Check if device hotplug event results in a device node being created.
-		if uEv.DevName != "" && strings.HasPrefix(uEv.DevPath, rootBusPath) {
+		if uEv.DevName != "" &&
+			(strings.HasPrefix(uEv.DevPath, rootBusPath) || strings.HasPrefix(uEv.DevPath, acpiDevPath)) {
 			// Lock is needed to safey read and modify the pciDeviceMap and deviceWatchers.
 			// This makes sure that watchers do not access the map while it is being updated.
 			s.Lock()
@@ -762,6 +763,11 @@ func (s *sandbox) listenToUdevEvents() {
 					goto OUT
 				}
 
+				// pmem/nvdimm case
+				if strings.Contains(uEv.DevPath, pfnDevPrefix) && strings.HasSuffix(uEv.DevPath, devAddress) {
+					goto OUT
+				}
+
 				if strings.Contains(uEv.DevPath, devAddress) {
 					// scsi driver case
 					if strings.HasSuffix(devAddress, scsiBlockSuffix) {

device.go

@@ -46,6 +46,7 @@ var (
 	pciBusPathFormat    = "%s/%s/pci_bus/"
 	systemDevPath       = "/dev"
 	getSCSIDevPath      = getSCSIDevPathImpl
+	getPmemDevPath      = getPmemDevPathImpl
 	getPCIDeviceName    = getPCIDeviceNameImpl
 	getDevicePCIAddress = getDevicePCIAddressImpl
 	scanSCSIBus         = scanSCSIBusImpl
@@ -359,6 +360,13 @@ func getSCSIDevPathImpl(s *sandbox, scsiAddr string) (string, error) {
 	return getDeviceName(s, devPath)
 }
 
+func getPmemDevPathImpl(s *sandbox, devPmemPath string) (string, error) {
+	// for example: /block/pmem1
+	devPath := filepath.Join("/", scsiBlockSuffix, filepath.Base(devPmemPath))
+
+	return getDeviceName(s, devPath)
+}
+
 // checkCCWBusFormat checks the format for the ccw bus. It needs to be in the form 0.<n>.<dddd>
 // n is the subchannel set ID - integer from 0 up to 3
 // dddd is the device id - integer in hex up to 0xffff

device_amd64.go

@@ -9,4 +9,15 @@ package main
 
 const (
 	rootBusPath = "/devices/pci0000:00"
+
+	// From https://www.kernel.org/doc/Documentation/acpi/namespace.txt
+	// The Linux kernel's core ACPI subsystem creates struct acpi_device
+	// objects for ACPI namespace objects representing devices, power resources
+	// processors, thermal zones. Those objects are exported to user space via
+	// sysfs as directories in the subtree under /sys/devices/LNXSYSTM:00
+	acpiDevPath = "/devices/LNXSYSTM"
+
+	// /dev/pmemX devices exported in the ACPI sysfs (/devices/LNXSYSTM) are
+	// in a subdirectory whose prefix is pfn (page frame number).
+	pfnDevPrefix = "/pfn"
 )

device_arm64.go

@@ -6,4 +6,17 @@
 
 package main
 
-const rootBusPath = "/devices/platform/4010000000.pcie/pci0000:00"
+const (
+	rootBusPath = "/devices/platform/4010000000.pcie/pci0000:00"
+
+	// From https://www.kernel.org/doc/Documentation/acpi/namespace.txt
+	// The Linux kernel's core ACPI subsystem creates struct acpi_device
+	// objects for ACPI namespace objects representing devices, power resources
+	// processors, thermal zones. Those objects are exported to user space via
+	// sysfs as directories in the subtree under /sys/devices/LNXSYSTM:00
+	acpiDevPath = "/devices/LNXSYSTM"
+
+	// /dev/pmemX devices exported in the ACPI sysfs (/devices/LNXSYSTM) are
+	// in a subdirectory whose prefix is pfn (page frame number).
+	pfnDevPrefix = "/pfn"
+)

device_ppc64le.go

@@ -9,4 +9,15 @@ package main
 
 const (
 	rootBusPath = "/devices/pci0000:00"
+
+	// From https://www.kernel.org/doc/Documentation/acpi/namespace.txt
+	// The Linux kernel's core ACPI subsystem creates struct acpi_device
+	// objects for ACPI namespace objects representing devices, power resources
+	// processors, thermal zones. Those objects are exported to user space via
+	// sysfs as directories in the subtree under /sys/devices/LNXSYSTM:00
+	acpiDevPath = "/devices/LNXSYSTM"
+
+	// /dev/pmemX devices exported in the ACPI sysfs (/devices/LNXSYSTM) are
+	// in a subdirectory whose prefix is pfn (page frame number).
+	pfnDevPrefix = "/pfn"
 )

device_s390x.go

@@ -9,4 +9,15 @@ package main
 
 const (
 	rootBusPath = "/devices/css0"
+
+	// From https://www.kernel.org/doc/Documentation/acpi/namespace.txt
+	// The Linux kernel's core ACPI subsystem creates struct acpi_device
+	// objects for ACPI namespace objects representing devices, power resources
+	// processors, thermal zones. Those objects are exported to user space via
+	// sysfs as directories in the subtree under /sys/devices/LNXSYSTM:00
+	acpiDevPath = "/devices/LNXSYSTM"
+
+	// /dev/pmemX devices exported in the ACPI sysfs (/devices/LNXSYSTM) are
+	// in a subdirectory whose prefix is pfn (page frame number).
+	pfnDevPrefix = "/pfn"
 )

mount.go

@@ -217,6 +217,7 @@ var storageHandlerList = map[string]storageHandler{
 	driverSCSIType:      virtioSCSIStorageHandler,
 	driverEphemeralType: ephemeralStorageHandler,
 	driverLocalType:     localStorageHandler,
+	driverNvdimmType:    nvdimmStorageHandler,
 }
 
 func ephemeralStorageHandler(_ context.Context, storage pb.Storage, s *sandbox) (string, error) {
@@ -303,6 +304,7 @@ func virtioBlkStorageHandler(_ context.Context, storage pb.Storage, s *sandbox)
 		FileInfo, err := os.Stat(storage.Source)
 		if err != nil {
 			return "", err
+
 		}
 		// Make sure the virt path is valid
 		if FileInfo.Mode()&os.ModeDevice == 0 {
@@ -321,6 +323,22 @@ func virtioBlkStorageHandler(_ context.Context, storage pb.Storage, s *sandbox)
 	return commonStorageHandler(storage)
 }
 
+func nvdimmStorageHandler(_ context.Context, storage pb.Storage, s *sandbox) (string, error) {
+	// waiting for a pmem device
+	if strings.HasPrefix(storage.Source, "/dev") && strings.HasPrefix(filepath.Base(storage.Source), "pmem") {
+		// Retrieve the device path from ACPI pmem address.
+		// for example: /devices/LNXSYSTM:00/LNXSYBUS:00/ACPI0012:00/ndbus0/region1/pfn1.1/block/pmem1
+		devPath, err := getPmemDevPath(s, storage.Source)
+		if err != nil {
+			return "", err
+		}
+		storage.Source = devPath
+		return commonStorageHandler(storage)
+	}
+
+	return "", fmt.Errorf("invalid nvdimm source path: %v", storage.Source)
+}
+
 // virtioSCSIStorageHandler handles the storage for scsi driver.
 func virtioSCSIStorageHandler(ctx context.Context, storage pb.Storage, s *sandbox) (string, error) {
 	// Retrieve the device path from SCSI address.

mount_test.go

@@ -265,6 +265,47 @@ func TestVirtioBlkStorageHandlerSuccessful(t *testing.T) {
 	assert.Nil(t, err, "storageBlockStorageDriverHandler() failed: %v", err)
 }
 
+func TestNvdimmStorageHandlerSuccessful(t *testing.T) {
+	skipUnlessRoot(t)
+
+	completePCIAddr := "/devices/LNXSYSTM/LNXSYBUS/ACPI/ndbus0/region1/pfn1.1/block/pmem0"
+	pmemDev := "/dev/pmem0"
+	devPath, err := createFakeDevicePath()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(devPath)
+
+	dirPath, err := ioutil.TempDir("", "fake-dir")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dirPath)
+
+	storage := pb.Storage{
+		Source:     pmemDev,
+		MountPoint: filepath.Join(dirPath, "test-mount"),
+	}
+	defer syscall.Unmount(storage.MountPoint, 0)
+
+	s := &sandbox{
+		pciDeviceMap: make(map[string]string),
+	}
+
+	s.Lock()
+	s.pciDeviceMap[completePCIAddr] = devPath
+	s.Unlock()
+
+	storage.Fstype = "bind"
+	storage.Options = []string{"rbind"}
+
+	ctx := context.Background()
+
+	systemDevPath = ""
+	_, err = nvdimmStorageHandler(ctx, storage, s)
+	assert.NoError(t, err)
+}
+
 func TestVirtioSCSIStorageHandlerFailure(t *testing.T) {
 	skipIfRoot(t)