persist: add interface for global read/write

Add two interfaces for fs storage driver for supporting global writing
and reading, which is used by ACRN.

Signed-off-by: Wei Zhang <[email protected]>

virtcontainers/acrn.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/kata-containers/runtime/pkg/rootless"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
+	"github.com/kata-containers/runtime/virtcontainers/persist"
 	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/runtime/virtcontainers/persist/fs"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/uuid"
@@ -41,7 +42,7 @@ const (
 // VMUUIDStoragePath is the uuid directory.
 // It will contain all uuid info used by guest vm.
 var VMUUIDStoragePath = func() string {
-	path := filepath.Join("/run/vc", UUIDPathSuffix)
+	path := filepath.Join(fs.StorageRootPath(), UUIDPathSuffix)
 	if rootless.IsRootless() {
 		return filepath.Join(rootless.GetRootlessDir(), path)
 	}
@@ -742,7 +743,7 @@ func (a *Acrn) GetACRNUUIDBytes(uid string) (uuid.UUID, error) {
 }
 
 // GetNextAvailableUUID returns next available UUID VM creation
-// If no validl UUIDs are available it returns err.
+// If no valid UUIDs are available it returns err.
 func (a *Acrn) GetNextAvailableUUID() (string, error) {
 	var MaxVMSupported uint8
 	var Idx uint8
@@ -796,78 +797,38 @@ func (a *Acrn) GetMaxSupportedACRNVM() (uint8, error) {
 }
 
 func (a *Acrn) storeInfo() error {
-	dirPath := VMUUIDStoragePath()
-
-	_, err := os.Stat(dirPath)
-	if os.IsNotExist(err) {
-		// Root directory
-		a.Logger().WithField("path", dirPath).Debugf("Creating UUID directory")
-		if err := os.MkdirAll(dirPath, DirMode); err != nil {
-			return err
-		}
-	} else if err != nil {
-		return err
-	}
-
-	dirf, err := os.Open(dirPath)
+	store, err := persist.GetDriver("fs")
 	if err != nil {
 		return err
 	}
-	defer dirf.Close()
-
-	if err := syscall.Flock(int(dirf.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
-		return err
-	}
-
-	// write data
-	f, err := os.OpenFile(filepath.Join(dirPath, uuidFile), os.O_RDWR|os.O_CREATE, 0755)
-	if err != nil {
-		return fmt.Errorf("failed to store information into uuid.json: %v", err)
-	}
-	defer f.Close()
+	relPath := filepath.Join(UUIDPathSuffix, uuidFile)
 
 	jsonOut, err := json.Marshal(a.info)
 	if err != nil {
 		return fmt.Errorf("Could not marshall data: %s", err)
 	}
-	f.Write(jsonOut)
+
+	if err := store.GlobalWrite(relPath, jsonOut); err != nil {
+		return fmt.Errorf("failed to write uuid to file: %v", err)
+	}
+
 	return nil
 }
 
 func (a *Acrn) loadInfo() error {
-	dirPath := VMUUIDStoragePath()
-
-	_, err := os.Stat(dirPath)
-	if err != nil {
-		return fmt.Errorf("failed to load ACRN information: %v", err)
-	}
-
-	dirf, err := os.Open(dirPath)
+	store, err := persist.GetDriver("fs")
 	if err != nil {
 		return err
 	}
+	relPath := filepath.Join(UUIDPathSuffix, uuidFile)
 
-	if err := syscall.Flock(int(dirf.Fd()), syscall.LOCK_SH|syscall.LOCK_NB); err != nil {
-		dirf.Close()
-		return err
-	}
-
-	defer dirf.Close()
-
-	// write data
-	f, err := os.Open(filepath.Join(dirPath, uuidFile))
+	data, err := store.GlobalRead(relPath)
 	if err != nil {
-		return fmt.Errorf("failed to load information into uuid.json: %v", err)
+		return fmt.Errorf("failed to read uuid from file: %v", err)
 	}
 
-	dec := json.NewDecoder(f)
-	if dec != nil {
-		return fmt.Errorf("failed to create json decoder")
-	}
-
-	err = dec.Decode(&a.info)
-	if err != nil {
-		return fmt.Errorf("could not decode data: %v", err)
+	if err := json.Unmarshal(data, &a.info); err != nil {
+		return fmt.Errorf("failed to unmarshal uuid info: %v", err)
 	}
 	return nil
 }

virtcontainers/api_test.go

@@ -1364,7 +1364,6 @@ func TestProcessListContainer(t *testing.T) {
 
 	pImpl, ok := p.(*Sandbox)
 	assert.True(ok)
-	// defer store.DeleteAll()
 
 	contConfig := newTestContainerConfigNoop(contID)
 	_, c, err := CreateContainer(ctx, p.ID(), contConfig)

virtcontainers/persist/api/interface.go

@@ -17,4 +17,12 @@ type PersistDriver interface {
 	// Lock locks the persist driver, "exclusive" decides whether the lock is exclusive or shared.
 	// It returns Unlock Function and errors
 	Lock(sid string, exclusive bool) (func() error, error)
+
+	// GlobalWrite writes "data" to "StorageRootPath"/"relativePath";
+	// GlobalRead reads "data" from "StorageRootPath"/"relativePath";
+	// these functions are used for writing/reading some global data,
+	// they are specially designed for ACRN so far.
+	// Don't use them too much unless you have no other choice! @weizhang555
+	GlobalWrite(relativePath string, data []byte) error
+	GlobalRead(relativePath string) ([]byte, error)
 }

virtcontainers/persist/fs/fs.go

@@ -23,10 +23,10 @@ import (
 const persistFile = "persist.json"
 
 // dirMode is the permission bits used for creating a directory
-const dirMode = os.FileMode(0700)
+const dirMode = os.FileMode(0700) | os.ModeDir
 
 // fileMode is the permission bits used for creating a file
-const fileMode = os.FileMode(0640)
+const fileMode = os.FileMode(0600)
 
 // storagePathSuffix is the suffix used for all storage paths
 //
@@ -40,25 +40,33 @@ const sandboxPathSuffix = "sbs"
 // vmPathSuffix is the suffix used for guest VMs.
 const vmPathSuffix = "vm"
 
-// RunStoragePath is the sandbox runtime directory.
-// It will contain one state.json and one lock file for each created sandbox.
-var RunStoragePath = func() string {
-	path := filepath.Join("/run", storagePathSuffix, sandboxPathSuffix)
+var StorageRootPath = func() string {
+	path := filepath.Join("/run", storagePathSuffix)
 	if rootless.IsRootless() {
 		return filepath.Join(rootless.GetRootlessDir(), path)
 	}
 	return path
 }
 
+// RunStoragePath is the sandbox runtime directory.
+// It will contain one state.json and one lock file for each created sandbox.
+var RunStoragePath = func() string {
+	return filepath.Join(StorageRootPath(), sandboxPathSuffix)
+}
+
 // RunVMStoragePath is the vm directory.
 // It will contain all guest vm sockets and shared mountpoints.
 // The function is declared this way for mocking in unit tests
 var RunVMStoragePath = func() string {
-	path := filepath.Join("/run", storagePathSuffix, vmPathSuffix)
-	if rootless.IsRootless() {
-		return filepath.Join(rootless.GetRootlessDir(), path)
+	return filepath.Join(StorageRootPath(), vmPathSuffix)
+}
+
+// TestSetRunStoragePath set RunStoragePath to path
+// this function is only used for testing purpose
+func TestSetRunStoragePath(path string) {
+	RunStoragePath = func() string {
+		return path
 	}
-	return path
 }
 
 // FS storage driver implementation
@@ -299,10 +307,57 @@ func (fs *FS) Lock(sandboxID string, exclusive bool) (func() error, error) {
 	return unlockFunc, nil
 }
 
-// TestSetRunStoragePath set RunStoragePath to path
-// this function is only used for testing purpose
-func TestSetRunStoragePath(path string) {
-	RunStoragePath = func() string {
-		return path
+func (fs *FS) GlobalWrite(relativePath string, data []byte) error {
+	path := filepath.Join(StorageRootPath(), relativePath)
+	path, err := filepath.Abs(filepath.Clean(path))
+	if err != nil {
+		return fmt.Errorf("failed to find abs path for %q: %v", relativePath, err)
+	}
+
+	dir := filepath.Dir(path)
+
+	_, err = os.Stat(dir)
+	if os.IsNotExist(err) {
+		if err := os.MkdirAll(dir, dirMode); err != nil {
+			fs.Logger().WithError(err).Errorf("failed to create dir %q", dir)
+			return err
+		}
+	} else if err != nil {
+		return err
+	}
+
+	f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, fileMode)
+	if err != nil {
+		fs.Logger().WithError(err).Errorf("failed to open file %q for writting", path)
+		return err
+	}
+	defer f.Close()
+
+	if _, err := f.Write(data); err != nil {
+		fs.Logger().WithError(err).Errorf("failed to write file %q: %v ", path, err)
+		return err
+	}
+	return nil
+}
+
+func (fs *FS) GlobalRead(relativePath string) ([]byte, error) {
+	path := filepath.Join(StorageRootPath(), relativePath)
+	path, err := filepath.Abs(filepath.Clean(path))
+	if err != nil {
+		return nil, fmt.Errorf("failed to find abs path for %q: %v", relativePath, err)
+	}
+
+	f, err := os.Open(path)
+	if err != nil {
+		fs.Logger().WithError(err).Errorf("failed to open file %q for reading", path)
+		return nil, err
+	}
+	defer f.Close()
+
+	data, err := ioutil.ReadAll(f)
+	if err != nil {
+		fs.Logger().WithError(err).Errorf("failed to read file %q: %v ", path, err)
+		return nil, err
 	}
+	return data, nil
 }

virtcontainers/persist/fs/fs_test.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"testing"
 
 	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
@@ -28,7 +29,27 @@ func getFsDriver() (*FS, error) {
 	return fs, nil
 }
 
+func initTestDir() func() {
+	testDir, _ := ioutil.TempDir("", "vc-tmp-")
+	// allow the tests to run without affecting the host system.
+	rootSave := StorageRootPath()
+	StorageRootPath = func() string {
+		return filepath.Join(testDir, "vc")
+	}
+
+	os.MkdirAll(testDir, dirMode)
+
+	return func() {
+		StorageRootPath = func() string {
+			return rootSave
+		}
+		os.RemoveAll(testDir)
+	}
+}
+
 func TestFsLockShared(t *testing.T) {
+	defer initTestDir()()
+
 	fs, err := getFsDriver()
 	assert.Nil(t, err)
 	assert.NotNil(t, fs)
@@ -61,6 +82,8 @@ func TestFsLockShared(t *testing.T) {
 }
 
 func TestFsLockExclusive(t *testing.T) {
+	defer initTestDir()()
+
 	fs, err := getFsDriver()
 	assert.Nil(t, err)
 	assert.NotNil(t, fs)
@@ -87,6 +110,8 @@ func TestFsLockExclusive(t *testing.T) {
 }
 
 func TestFsDriver(t *testing.T) {
+	defer initTestDir()()
+
 	fs, err := getFsDriver()
 	assert.Nil(t, err)
 	assert.NotNil(t, fs)
@@ -163,3 +188,25 @@ func TestFsDriver(t *testing.T) {
 	assert.NotNil(t, err)
 	assert.True(t, os.IsNotExist(err))
 }
+
+func TestGlobalReadWrite(t *testing.T) {
+	defer initTestDir()()
+
+	relPath := "test/123/aaa.json"
+	data := "hello this is testing global read write"
+
+	fs, err := getFsDriver()
+	assert.Nil(t, err)
+	assert.NotNil(t, fs)
+
+	err = fs.GlobalWrite(relPath, []byte(data))
+	assert.Nil(t, err)
+
+	out, err := fs.GlobalRead(relPath)
+	assert.Nil(t, err)
+	assert.Equal(t, string(out), data)
+
+	out, err = fs.GlobalRead("nonexist")
+	assert.NotNil(t, err)
+	assert.Nil(t, out)
+}

virtcontainers/sandbox.go

@@ -547,7 +547,9 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor
 	s.devManager = deviceManager.NewDeviceManager(sandboxConfig.HypervisorConfig.BlockDeviceDriver, nil)
 
 	// Ignore the error. Restore can fail for a new sandbox
-	s.Restore()
+	if err := s.Restore(); err != nil {
+		s.Logger().WithError(err).Debug("restore sandbox failed")
+	}
 
 	// new store doesn't require hypervisor to be stored immediately
 	if err = s.hypervisor.createSandbox(ctx, s.id, s.networkNS, &sandboxConfig.HypervisorConfig, s.stateful); err != nil {