vc: add rootless dir to path variables

Modify some path variables to be functions that return the path
with the rootless directory prefix if running rootlessly.

Fixes: #1827

Signed-off-by: Gabi Beyer <[email protected]>

virtcontainers/acrn.go

@@ -370,7 +370,7 @@ func (a *acrn) startSandbox(timeoutSecs int) error {
 		a.Logger().WithField("default-kernel-parameters", formatted).Debug()
 	}
 
-	vmPath := filepath.Join(store.RunVMStoragePath, a.id)
+	vmPath := filepath.Join(store.RunVMStoragePath(), a.id)
 	err := os.MkdirAll(vmPath, store.DirMode)
 	if err != nil {
 		return err
@@ -574,7 +574,7 @@ func (a *acrn) getSandboxConsole(id string) (string, error) {
 	span, _ := a.trace("getSandboxConsole")
 	defer span.Finish()
 
-	return utils.BuildSocketPath(store.RunVMStoragePath, id, acrnConsoleSocket)
+	return utils.BuildSocketPath(store.RunVMStoragePath(), id, acrnConsoleSocket)
 }
 
 func (a *acrn) saveSandbox() error {

virtcontainers/acrn_test.go

@@ -199,7 +199,7 @@ func TestAcrnGetSandboxConsole(t *testing.T) {
 		ctx: context.Background(),
 	}
 	sandboxID := "testSandboxID"
-	expected := filepath.Join(store.RunVMStoragePath, sandboxID, consoleSocket)
+	expected := filepath.Join(store.RunVMStoragePath(), sandboxID, consoleSocket)
 
 	result, err := a.getSandboxConsole(sandboxID)
 	assert.NoError(err)

virtcontainers/api.go

@@ -307,7 +307,7 @@ func ListSandbox(ctx context.Context) ([]SandboxStatus, error) {
 	span, ctx := trace(ctx, "ListSandbox")
 	defer span.Finish()
 
-	dir, err := os.Open(store.ConfigStoragePath)
+	dir, err := os.Open(store.ConfigStoragePath())
 	if err != nil {
 		if os.IsNotExist(err) {
 			// No sandbox directory is not an error

virtcontainers/container.go

@@ -1103,7 +1103,7 @@ func (c *Container) stop(force bool) error {
 		return err
 	}
 
-	if err := bindUnmountContainerRootfs(c.ctx, kataHostSharedDir, c.sandbox.id, c.id); err != nil && !force {
+	if err := bindUnmountContainerRootfs(c.ctx, kataHostSharedDir(), c.sandbox.id, c.id); err != nil && !force {
 		return err
 	}
 
@@ -1350,7 +1350,7 @@ func (c *Container) plugDevice(devicePath string) error {
 	if c.checkBlockDeviceSupport() && stat.Mode&unix.S_IFBLK == unix.S_IFBLK {
 		b, err := c.sandbox.devManager.NewDevice(config.DeviceInfo{
 			HostPath:      devicePath,
-			ContainerPath: filepath.Join(kataGuestSharedDir, c.id),
+			ContainerPath: filepath.Join(kataGuestSharedDir(), c.id),
 			DevType:       "b",
 			Major:         int64(unix.Major(stat.Rdev)),
 			Minor:         int64(unix.Minor(stat.Rdev)),

virtcontainers/hypervisor.go

@@ -688,7 +688,7 @@ func generateVMSocket(id string, useVsock bool) (interface{}, error) {
 		}, nil
 	}
 
-	path, err := utils.BuildSocketPath(filepath.Join(store.RunVMStoragePath, id), defaultSocketName)
+	path, err := utils.BuildSocketPath(filepath.Join(store.RunVMStoragePath(), id), defaultSocketName)
 	if err != nil {
 		return nil, err
 	}

virtcontainers/kata_agent.go

@@ -23,6 +23,7 @@ import (
 	aTypes "github.com/kata-containers/agent/pkg/types"
 	kataclient "github.com/kata-containers/agent/protocols/client"
 	"github.com/kata-containers/agent/protocols/grpc"
+	"github.com/kata-containers/runtime/pkg/rootless"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
 	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
@@ -56,10 +57,10 @@ var (
 	defaultRequestTimeout       = 60 * time.Second
 	errorMissingProxy           = errors.New("Missing proxy pointer")
 	errorMissingOCISpec         = errors.New("Missing OCI specification")
-	kataHostSharedDir           = "/run/kata-containers/shared/sandboxes/"
-	kataGuestSharedDir          = "/run/kata-containers/shared/containers/"
+	defaultKataHostSharedDir    = "/run/kata-containers/shared/sandboxes/"
+	defaultKataGuestSharedDir   = "/run/kata-containers/shared/containers/"
 	mountGuest9pTag             = "kataShared"
-	kataGuestSandboxDir         = "/run/kata-containers/sandbox/"
+	defaultKataGuestSandboxDir  = "/run/kata-containers/sandbox/"
 	type9pFs                    = "9p"
 	typeVirtioFS                = "virtio_fs"
 	typeVirtioFSNoCache         = "none"
@@ -71,11 +72,11 @@ var (
 	kataNvdimmDevType           = "nvdimm"
 	kataVirtioFSDevType         = "virtio-fs"
 	sharedDir9pOptions          = []string{"trans=virtio,version=9p2000.L,cache=mmap", "nodev"}
-	sharedDirVirtioFSOptions    = []string{"default_permissions,allow_other,rootmode=040000,user_id=0,group_id=0,tag=" + mountGuest9pTag, "nodev"}
+	sharedDirVirtioFSOptions    = []string{"default_permissions,allow_other,rootmode=040000,user_id=0,group_id=0,dax,tag=" + mountGuest9pTag, "nodev"}
 	sharedDirVirtioFSDaxOptions = "dax"
 	shmDir                      = "shm"
 	kataEphemeralDevType        = "ephemeral"
-	ephemeralPath               = filepath.Join(kataGuestSandboxDir, kataEphemeralDevType)
+	defaultEphemeralPath        = filepath.Join(defaultKataGuestSandboxDir, kataEphemeralDevType)
 	grpcMaxDataSize             = int64(1024 * 1024)
 	localDirOptions             = []string{"mode=0777"}
 	maxHostnameLen              = 64
@@ -124,6 +125,40 @@ const (
 	grpcStopTracingRequest       = "grpc.StopTracingRequest"
 )
 
+// The function is declared this way for mocking in unit tests
+var kataHostSharedDir = func() string {
+	if rootless.IsRootless() {
+		// filepath.Join removes trailing slashes, but it is necessary for mounting
+		return filepath.Join(rootless.GetRootlessDir(), defaultKataHostSharedDir) + "/"
+	}
+	return defaultKataHostSharedDir
+}
+
+// The function is declared this way for mocking in unit tests
+var kataGuestSharedDir = func() string {
+	if rootless.IsRootless() {
+		// filepath.Join removes trailing slashes, but it is necessary for mounting
+		return filepath.Join(rootless.GetRootlessDir(), defaultKataGuestSharedDir) + "/"
+	}
+	return defaultKataGuestSharedDir
+}
+
+// The function is declared this way for mocking in unit tests
+var kataGuestSandboxDir = func() string {
+	if rootless.IsRootless() {
+		// filepath.Join removes trailing slashes, but it is necessary for mounting
+		return filepath.Join(rootless.GetRootlessDir(), defaultKataGuestSandboxDir) + "/"
+	}
+	return defaultKataGuestSandboxDir
+}
+
+func ephemeralPath() string {
+	if rootless.IsRootless() {
+		return filepath.Join(kataGuestSandboxDir(), kataEphemeralDevType)
+	}
+	return defaultEphemeralPath
+}
+
 // KataAgentConfig is a structure storing information needed
 // to reach the Kata Containers agent.
 type KataAgentConfig struct {
@@ -182,11 +217,11 @@ func (k *kataAgent) Logger() *logrus.Entry {
 }
 
 func (k *kataAgent) getVMPath(id string) string {
-	return filepath.Join(store.RunVMStoragePath, id)
+	return filepath.Join(store.RunVMStoragePath(), id)
 }
 
 func (k *kataAgent) getSharePath(id string) string {
-	return filepath.Join(kataHostSharedDir, id)
+	return filepath.Join(kataHostSharedDir(), id)
 }
 
 // KataAgentSetDefaultTraceConfigOptions validates agent trace options and
@@ -838,7 +873,7 @@ func setupStorages(sandbox *Sandbox) []*grpc.Storage {
 			sharedVolume := &grpc.Storage{
 				Driver:     kataVirtioFSDevType,
 				Source:     "none",
-				MountPoint: kataGuestSharedDir,
+				MountPoint: kataGuestSharedDir(),
 				Fstype:     typeVirtioFS,
 				Options:    sharedDirVirtioFSOptions,
 			}
@@ -850,7 +885,7 @@ func setupStorages(sandbox *Sandbox) []*grpc.Storage {
 			sharedVolume := &grpc.Storage{
 				Driver:     kata9pDevType,
 				Source:     mountGuest9pTag,
-				MountPoint: kataGuestSharedDir,
+				MountPoint: kataGuestSharedDir(),
 				Fstype:     type9pFs,
 				Options:    sharedDir9pOptions,
 			}
@@ -860,7 +895,7 @@ func setupStorages(sandbox *Sandbox) []*grpc.Storage {
 	}
 
 	if sandbox.shmSize > 0 {
-		path := filepath.Join(kataGuestSandboxDir, shmDir)
+		path := filepath.Join(kataGuestSandboxDir(), shmDir)
 		shmSizeOption := fmt.Sprintf("size=%d", sandbox.shmSize)
 
 		shmStorage := &grpc.Storage{
@@ -970,7 +1005,7 @@ func (k *kataAgent) replaceOCIMountsForStorages(spec *specs.Spec, volumeStorages
 			// Create a temporary location to mount the Storage. Mounting to the correct location
 			// will be handled by the OCI mount structure.
 			filename := fmt.Sprintf("%s-%s", uuid.Generate().String(), filepath.Base(m.Destination))
-			path := filepath.Join(kataGuestSharedDir, filename)
+			path := filepath.Join(kataGuestSharedDir(), filename)
 
 			k.Logger().Debugf("Replacing OCI mount source (%s) with %s", m.Source, path)
 			ociMounts[index].Source = path
@@ -1052,7 +1087,7 @@ func (k *kataAgent) handleShm(grpcSpec *grpc.Spec, sandbox *Sandbox) {
 		if sandbox.shmSize > 0 {
 			grpcSpec.Mounts[idx].Type = "bind"
 			grpcSpec.Mounts[idx].Options = []string{"rbind"}
-			grpcSpec.Mounts[idx].Source = filepath.Join(kataGuestSandboxDir, shmDir)
+			grpcSpec.Mounts[idx].Source = filepath.Join(kataGuestSandboxDir(), shmDir)
 			k.Logger().WithField("shm-size", sandbox.shmSize).Info("Using sandbox shm")
 		} else {
 			sizeOption := fmt.Sprintf("size=%d", DefaultShmSize)
@@ -1121,7 +1156,7 @@ func (k *kataAgent) rollbackFailingContainerCreation(c *Container) {
 			k.Logger().WithError(err2).Error("rollback failed unmountHostMounts()")
 		}
 
-		if err2 := bindUnmountContainerRootfs(k.ctx, kataHostSharedDir, c.sandbox.id, c.id); err2 != nil {
+		if err2 := bindUnmountContainerRootfs(k.ctx, kataHostSharedDir(), c.sandbox.id, c.id); err2 != nil {
 			k.Logger().WithError(err2).Error("rollback failed bindUnmountContainerRootfs()")
 		}
 	}
@@ -1189,7 +1224,7 @@ func (k *kataAgent) buildContainerRootfs(sandbox *Sandbox, c *Container, rootPat
 	// (kataGuestSharedDir) is already mounted in the
 	// guest. We only need to mount the rootfs from
 	// the host and it will show up in the guest.
-	if err := bindMountContainerRootfs(k.ctx, kataHostSharedDir, sandbox.id, c.id, c.rootFs.Target, false); err != nil {
+	if err := bindMountContainerRootfs(k.ctx, kataHostSharedDir(), sandbox.id, c.id, c.rootFs.Target, false); err != nil {
 		return nil, err
 	}
 
@@ -1215,7 +1250,7 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process,
 	var rootfs *grpc.Storage
 
 	// This is the guest absolute root path for that container.
-	rootPathParent := filepath.Join(kataGuestSharedDir, c.id)
+	rootPathParent := filepath.Join(kataGuestSharedDir(), c.id)
 	rootPath := filepath.Join(rootPathParent, c.rootfsSuffix)
 
 	// In case the container creation fails, the following defer statement
@@ -1242,7 +1277,7 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process,
 	}
 
 	// Handle container mounts
-	newMounts, ignoredMounts, err := c.mountSharedDirMounts(kataHostSharedDir, kataGuestSharedDir)
+	newMounts, ignoredMounts, err := c.mountSharedDirMounts(kataHostSharedDir(), kataGuestSharedDir())
 	if err != nil {
 		return nil, err
 	}
@@ -1345,7 +1380,7 @@ func (k *kataAgent) handleEphemeralStorage(mounts []specs.Mount) []*grpc.Storage
 	for idx, mnt := range mounts {
 		if mnt.Type == KataEphemeralDevType {
 			// Set the mount source path to a path that resides inside the VM
-			mounts[idx].Source = filepath.Join(ephemeralPath, filepath.Base(mnt.Source))
+			mounts[idx].Source = filepath.Join(ephemeralPath(), filepath.Base(mnt.Source))
 			// Set the mount type to "bind"
 			mounts[idx].Type = "bind"
 
@@ -1374,7 +1409,7 @@ func (k *kataAgent) handleLocalStorage(mounts []specs.Mount, sandboxID string, r
 			// We rely on the fact that the first container in the VM has the same ID as the sandbox ID.
 			// In Kubernetes, this is usually the pause container and we depend on it existing for
 			// local directories to work.
-			mounts[idx].Source = filepath.Join(kataGuestSharedDir, sandboxID, rootfsSuffix, KataLocalDevType, filepath.Base(mnt.Source))
+			mounts[idx].Source = filepath.Join(kataGuestSharedDir(), sandboxID, rootfsSuffix, KataLocalDevType, filepath.Base(mnt.Source))
 
 			// Create a storage struct so that the kata agent is able to create the
 			// directory inside the VM.

virtcontainers/kata_agent_test.go

@@ -6,6 +6,7 @@
 package virtcontainers
 
 import (
+	"bufio"
 	"context"
 	"fmt"
 	"io/ioutil"
@@ -25,6 +26,7 @@ import (
 
 	aTypes "github.com/kata-containers/agent/pkg/types"
 	pb "github.com/kata-containers/agent/protocols/grpc"
+	"github.com/kata-containers/runtime/pkg/rootless"
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/runtime/virtcontainers/device/drivers"
@@ -359,7 +361,7 @@ func TestHandleEphemeralStorage(t *testing.T) {
 	epheStorages := k.handleEphemeralStorage(ociMounts)
 
 	epheMountPoint := epheStorages[0].GetMountPoint()
-	expected := filepath.Join(ephemeralPath, filepath.Base(mountSource))
+	expected := filepath.Join(ephemeralPath(), filepath.Base(mountSource))
 	assert.Equal(t, epheMountPoint, expected,
 		"Ephemeral mount point didn't match: got %s, expecting %s", epheMountPoint, expected)
 }
@@ -384,7 +386,7 @@ func TestHandleLocalStorage(t *testing.T) {
 	assert.Equal(t, len(localStorages), 1)
 
 	localMountPoint := localStorages[0].GetMountPoint()
-	expected := filepath.Join(kataGuestSharedDir, sandboxID, rootfsSuffix, KataLocalDevType, filepath.Base(mountSource))
+	expected := filepath.Join(kataGuestSharedDir(), sandboxID, rootfsSuffix, KataLocalDevType, filepath.Base(mountSource))
 	assert.Equal(t, localMountPoint, expected)
 }
 
@@ -531,7 +533,7 @@ func TestHandleShm(t *testing.T) {
 	assert.NotEmpty(g.Mounts[0].Destination)
 	assert.Equal(g.Mounts[0].Destination, "/dev/shm")
 	assert.Equal(g.Mounts[0].Type, "bind")
-	assert.NotEmpty(g.Mounts[0].Source, filepath.Join(kataGuestSharedDir, shmDir))
+	assert.NotEmpty(g.Mounts[0].Source, filepath.Join(kataGuestSharedDir(), shmDir))
 	assert.Equal(g.Mounts[0].Options, []string{"rbind"})
 
 	sandbox.shmSize = 0
@@ -896,15 +898,18 @@ func TestKataCleanupSandbox(t *testing.T) {
 	assert := assert.New(t)
 
 	kataHostSharedDirSaved := kataHostSharedDir
-	kataHostSharedDir, _ = ioutil.TempDir("", "kata-cleanup")
+	kataHostSharedDir = func() string {
+		td, _ := ioutil.TempDir("", "kata-cleanup")
+		return td
+	}
 	defer func() {
 		kataHostSharedDir = kataHostSharedDirSaved
 	}()
 
 	s := Sandbox{
 		id: "testFoo",
 	}
-	dir := path.Join(kataHostSharedDir, s.id)
+	dir := path.Join(kataHostSharedDir(), s.id)
 	err := os.MkdirAll(dir, 0777)
 	assert.Nil(err)
 
@@ -1110,3 +1115,29 @@ func TestKataAgentSetDefaultTraceConfigOptions(t *testing.T) {
 		}
 	}
 }
+
+func TestKataAgentDirs(t *testing.T) {
+	assert := assert.New(t)
+
+	uidmapFile, err := os.OpenFile("/proc/self/uid_map", os.O_RDONLY, 0)
+	assert.NoError(err)
+
+	line, err := bufio.NewReader(uidmapFile).ReadBytes('\n')
+	assert.NoError(err)
+
+	uidmap := strings.Fields(string(line))
+	expectedRootless := (uidmap[0] == "0" && uidmap[1] != "0")
+	assert.Equal(expectedRootless, rootless.IsRootless())
+
+	if expectedRootless {
+		assert.Equal(kataHostSharedDir(), os.Getenv("XDG_RUNTIME_DIR")+defaultKataHostSharedDir)
+		assert.Equal(kataGuestSharedDir(), os.Getenv("XDG_RUNTIME_DIR")+defaultKataGuestSharedDir)
+		assert.Equal(kataGuestSandboxDir(), os.Getenv("XDG_RUNTIME_DIR")+defaultKataGuestSandboxDir)
+		assert.Equal(ephemeralPath(), os.Getenv("XDG_RUNTIME_DIR")+defaultEphemeralPath)
+	} else {
+		assert.Equal(kataHostSharedDir(), defaultKataHostSharedDir)
+		assert.Equal(kataGuestSharedDir(), defaultKataGuestSharedDir)
+		assert.Equal(kataGuestSandboxDir(), defaultKataGuestSandboxDir)
+		assert.Equal(ephemeralPath(), defaultEphemeralPath)
+	}
+}