virtcontainers: add sandbox hotplug network API

Add sandbox hotplug network API to meet design Signed-off-by: Ruidong Cao <[email protected]>
egernst · Aug 16, 2018 · 1a17200 · 1a17200
1 parent 6666426
commit 1a17200
Show file tree

Hide file tree

Showing 13 changed files with 575 additions and 30 deletions.
diff --git a/virtcontainers/agent.go b/virtcontainers/agent.go
@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"syscall"
 
+	"github.com/kata-containers/agent/protocols/grpc"
 	"github.com/mitchellh/mapstructure"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 )
@@ -221,4 +222,16 @@ type agent interface {
 
 	// reseedRNG will reseed the guest random number generator
 	reseedRNG(data []byte) error
+
+	// updateInterface will tell the agent to update a nic for an existed Sandbox.
+	updateInterface(inf *grpc.Interface) (*grpc.Interface, error)
+
+	// listInterfaces will tell the agent to list interfaces of an existed Sandbox
+	listInterfaces() ([]*grpc.Interface, error)
+
+	// updateRoutes will tell the agent to update route table for an existed Sandbox.
+	updateRoutes(routes []*grpc.Route) ([]*grpc.Route, error)
+
+	// listRoutes will tell the agent to list routes of an existed Sandbox
+	listRoutes() ([]*grpc.Route, error)
 }
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
@@ -10,6 +10,7 @@ import (
 	"runtime"
 	"syscall"
 
+	"github.com/kata-containers/agent/protocols/grpc"
 	deviceApi "github.com/kata-containers/runtime/virtcontainers/device/api"
 	deviceConfig "github.com/kata-containers/runtime/virtcontainers/device/config"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
@@ -782,3 +783,93 @@ func AddDevice(sandboxID string, info deviceConfig.DeviceInfo) (deviceApi.Device
 
 	return s.AddDevice(info)
 }
+
+func toggleInterface(sandboxID string, inf *grpc.Interface, add bool) (*grpc.Interface, error) {
+	if sandboxID == "" {
+		return nil, errNeedSandboxID
+	}
+
+	lockFile, err := rwLockSandbox(sandboxID)
+	if err != nil {
+		return nil, err
+	}
+	defer unlockSandbox(lockFile)
+
+	s, err := fetchSandbox(sandboxID)
+	if err != nil {
+		return nil, err
+	}
+	if add {
+		return s.AddInterface(inf)
+	}
+	return s.RemoveInterface(inf)
+}
+
+// AddInterface is the virtcontainers add interface entry point.
+func AddInterface(sandboxID string, inf *grpc.Interface) (*grpc.Interface, error) {
+	return toggleInterface(sandboxID, inf, true)
+}
+
+// RemoveInterface is the virtcontainers remove interface entry point.
+func RemoveInterface(sandboxID string, inf *grpc.Interface) (*grpc.Interface, error) {
+	return toggleInterface(sandboxID, inf, false)
+}
+
+// ListInterfaces is the virtcontainers list interfaces entry point.
+func ListInterfaces(sandboxID string) ([]*grpc.Interface, error) {
+	if sandboxID == "" {
+		return nil, errNeedSandboxID
+	}
+
+	lockFile, err := rLockSandbox(sandboxID)
+	if err != nil {
+		return nil, err
+	}
+	defer unlockSandbox(lockFile)
+
+	s, err := fetchSandbox(sandboxID)
+	if err != nil {
+		return nil, err
+	}
+
+	return s.ListInterfaces()
+}
+
+// UpdateRoutes is the virtcontainers update routes entry point.
+func UpdateRoutes(sandboxID string, routes []*grpc.Route) ([]*grpc.Route, error) {
+	if sandboxID == "" {
+		return nil, errNeedSandboxID
+	}
+
+	lockFile, err := rwLockSandbox(sandboxID)
+	if err != nil {
+		return nil, err
+	}
+	defer unlockSandbox(lockFile)
+
+	s, err := fetchSandbox(sandboxID)
+	if err != nil {
+		return nil, err
+	}
+	return s.UpdateRoutes(routes)
+}
+
+// ListRoutes is the virtcontainers list routes entry point.
+func ListRoutes(sandboxID string) ([]*grpc.Route, error) {
+	if sandboxID == "" {
+		return nil, errNeedSandboxID
+	}
+
+	lockFile, err := rLockSandbox(sandboxID)
+	if err != nil {
+		return nil, err
+	}
+	defer unlockSandbox(lockFile)
+
+	s, err := fetchSandbox(sandboxID)
+	if err != nil {
+		return nil, err
+	}
+
+	return s.ListRoutes()
+}
diff --git a/virtcontainers/hyperstart_agent.go b/virtcontainers/hyperstart_agent.go
@@ -18,6 +18,7 @@ import (
 	"github.com/vishvananda/netlink"
 
 	proxyClient "github.com/clearcontainers/proxy/client"
+	"github.com/kata-containers/agent/protocols/grpc"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/hyperstart"
 	ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter"
@@ -852,6 +853,26 @@ func (h *hyper) onlineCPUMem(cpus uint32) error {
 	return nil
 }
 
+func (h *hyper) updateInterface(inf *grpc.Interface) (*grpc.Interface, error) {
+	// hyperstart-agent does not support update interface
+	return nil, nil
+}
+
+func (h *hyper) listInterfaces() ([]*grpc.Interface, error) {
+	// hyperstart-agent does not support list interfaces
+	return nil, nil
+}
+
+func (h *hyper) updateRoutes(routes []*grpc.Route) ([]*grpc.Route, error) {
+	// hyperstart-agent does not support update routes
+	return nil, nil
+}
+
+func (h *hyper) listRoutes() ([]*grpc.Route, error) {
+	// hyperstart-agent does not support list routes
+	return nil, nil
+}
+
 func (h *hyper) check() error {
 	// hyperstart-agent does not support check
 	return nil

diff --git a/virtcontainers/implementation.go b/virtcontainers/implementation.go
@@ -12,6 +12,7 @@ package virtcontainers
 import (
 	"syscall"
 
+	"github.com/kata-containers/agent/protocols/grpc"
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
@@ -148,3 +149,28 @@ func (impl *VCImpl) ResumeContainer(sandboxID, containerID string) error {
 func (impl *VCImpl) AddDevice(sandboxID string, info config.DeviceInfo) (api.Device, error) {
 	return AddDevice(sandboxID, info)
 }
+
+// AddInterface implements the VC function of the same name.
+func (impl *VCImpl) AddInterface(sandboxID string, inf *grpc.Interface) (*grpc.Interface, error) {
+	return AddInterface(sandboxID, inf)
+}
+
+// RemoveInterface implements the VC function of the same name.
+func (impl *VCImpl) RemoveInterface(sandboxID string, inf *grpc.Interface) (*grpc.Interface, error) {
+	return RemoveInterface(sandboxID, inf)
+}
+
+// ListInterfaces implements the VC function of the same name.
+func (impl *VCImpl) ListInterfaces(sandboxID string) ([]*grpc.Interface, error) {
+	return ListInterfaces(sandboxID)
+}
+
+// UpdateRoutes implements the VC function of the same name.
+func (impl *VCImpl) UpdateRoutes(sandboxID string, routes []*grpc.Route) ([]*grpc.Route, error) {
+	return UpdateRoutes(sandboxID, routes)
+}
+
+// ListRoutes implements the VC function of the same name.
+func (impl *VCImpl) ListRoutes(sandboxID string) ([]*grpc.Route, error) {
+	return ListRoutes(sandboxID)
+}
diff --git a/virtcontainers/interfaces.go b/virtcontainers/interfaces.go
@@ -9,6 +9,7 @@ import (
 	"io"
 	"syscall"
 
+	"github.com/kata-containers/agent/protocols/grpc"
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
@@ -45,6 +46,12 @@ type VC interface {
 	ResumeContainer(sandboxID, containerID string) error
 
 	AddDevice(sandboxID string, info config.DeviceInfo) (api.Device, error)
+
+	AddInterface(sandboxID string, inf *grpc.Interface) (*grpc.Interface, error)
+	RemoveInterface(sandboxID string, inf *grpc.Interface) (*grpc.Interface, error)
+	ListInterfaces(sandboxID string) ([]*grpc.Interface, error)
+	UpdateRoutes(sandboxID string, routes []*grpc.Route) ([]*grpc.Route, error)
+	ListRoutes(sandboxID string) ([]*grpc.Route, error)
 }
 
 // VCSandbox is the Sandbox interface
@@ -76,6 +83,12 @@ type VCSandbox interface {
 	IOStream(containerID, processID string) (io.WriteCloser, io.Reader, io.Reader, error)
 
 	AddDevice(info config.DeviceInfo) (api.Device, error)
+
+	AddInterface(inf *grpc.Interface) (*grpc.Interface, error)
+	RemoveInterface(inf *grpc.Interface) (*grpc.Interface, error)
+	ListInterfaces() ([]*grpc.Interface, error)
+	UpdateRoutes(routes []*grpc.Route) ([]*grpc.Route, error)
+	ListRoutes() ([]*grpc.Route, error)
 }
 
 // VCContainer is the Container interface

diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
@@ -440,6 +440,82 @@ func (k *kataAgent) generateInterfacesAndRoutes(networkNS NetworkNamespace) ([]*
 	return ifaces, routes, nil
 }
 
+func (k *kataAgent) updateInterface(ifc *grpc.Interface) (*grpc.Interface, error) {
+	// send update interface request
+	ifcReq := &grpc.UpdateInterfaceRequest{
+		Interface: ifc,
+	}
+	resultingInterface, err := k.sendReq(ifcReq)
+	if err != nil {
+		k.Logger().WithFields(logrus.Fields{
+			"interface-requested": fmt.Sprintf("%+v", ifc),
+			"resulting-interface": fmt.Sprintf("%+v", resultingInterface),
+		}).WithError(err).Error("update interface request failed")
+	}
+	if resultInterface, ok := resultingInterface.(*grpc.Interface); ok {
+		return resultInterface, err
+	}
+	return nil, err
+}
+
+func (k *kataAgent) updateInterfaces(interfaces []*grpc.Interface) error {
+	for _, ifc := range interfaces {
+		if _, err := k.updateInterface(ifc); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (k *kataAgent) updateRoutes(routes []*grpc.Route) ([]*grpc.Route, error) {
+	if routes != nil {
+		routesReq := &grpc.UpdateRoutesRequest{
+			Routes: &grpc.Routes{
+				Routes: routes,
+			},
+		}
+		resultingRoutes, err := k.sendReq(routesReq)
+		if err != nil {
+			k.Logger().WithFields(logrus.Fields{
+				"routes-requested": fmt.Sprintf("%+v", routes),
+				"resulting-routes": fmt.Sprintf("%+v", resultingRoutes),
+			}).WithError(err).Error("update routes request failed")
+		}
+		resultRoutes, ok := resultingRoutes.(*grpc.Routes)
+		if ok && resultRoutes != nil {
+			return resultRoutes.Routes, err
+		}
+		return nil, err
+	}
+	return nil, nil
+}
+
+func (k *kataAgent) listInterfaces() ([]*grpc.Interface, error) {
+	req := &grpc.ListInterfacesRequest{}
+	resultingInterfaces, err := k.sendReq(req)
+	if err != nil {
+		return nil, err
+	}
+	resultInterfaces, ok := resultingInterfaces.(*grpc.Interfaces)
+	if ok {
+		return resultInterfaces.Interfaces, err
+	}
+	return nil, err
+}
+
+func (k *kataAgent) listRoutes() ([]*grpc.Route, error) {
+	req := &grpc.ListRoutesRequest{}
+	resultingRoutes, err := k.sendReq(req)
+	if err != nil {
+		return nil, err
+	}
+	resultRoutes, ok := resultingRoutes.(*grpc.Routes)
+	if ok {
+		return resultRoutes.Routes, err
+	}
+	return nil, err
+}
+
 func (k *kataAgent) startProxy(sandbox *Sandbox) error {
 	var err error
 
@@ -515,36 +591,11 @@ func (k *kataAgent) startSandbox(sandbox *Sandbox) error {
 	if err != nil {
 		return err
 	}
-	for _, ifc := range interfaces {
-		// send update interface request
-		ifcReq := &grpc.UpdateInterfaceRequest{
-			Interface: ifc,
-		}
-		resultingInterface, err := k.sendReq(ifcReq)
-		if err != nil {
-			k.Logger().WithFields(logrus.Fields{
-				"interface-requested": fmt.Sprintf("%+v", ifc),
-				"resulting-interface": fmt.Sprintf("%+v", resultingInterface),
-			}).WithError(err).Error("update interface request failed")
-			return err
-		}
+	if err := k.updateInterfaces(interfaces); err != nil {
+		return err
 	}
-
-	if routes != nil {
-		routesReq := &grpc.UpdateRoutesRequest{
-			Routes: &grpc.Routes{
-				Routes: routes,
-			},
-		}
-
-		resultingRoutes, err := k.sendReq(routesReq)
-		if err != nil {
-			k.Logger().WithFields(logrus.Fields{
-				"routes-requested": fmt.Sprintf("%+v", routes),
-				"resulting-routes": fmt.Sprintf("%+v", resultingRoutes),
-			}).WithError(err).Error("update routes request failed")
-			return err
-		}
+	if _, err := k.updateRoutes(routes); err != nil {
+		return err
 	}
 
 	sharedDir9pOptions = append(sharedDir9pOptions, fmt.Sprintf("msize=%d", sandbox.config.HypervisorConfig.Msize9p))
@@ -1335,6 +1386,12 @@ func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) {
 	k.reqHandlers["grpc.UpdateInterfaceRequest"] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) {
 		return k.client.UpdateInterface(ctx, req.(*grpc.UpdateInterfaceRequest), opts...)
 	}
+	k.reqHandlers["grpc.ListInterfacesRequest"] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) {
+		return k.client.ListInterfaces(ctx, req.(*grpc.ListInterfacesRequest), opts...)
+	}
+	k.reqHandlers["grpc.ListRoutesRequest"] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) {
+		return k.client.ListRoutes(ctx, req.(*grpc.ListRoutesRequest), opts...)
+	}
 	k.reqHandlers["grpc.OnlineCPUMemRequest"] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) {
 		return k.client.OnlineCPUMem(ctx, req.(*grpc.OnlineCPUMemRequest), opts...)
 	}