virtcontainers: move cpu cgroup implementation

cpu cgroups are container's specific hence all containers even the sandbox
should be able o create, delete and update their cgroups. The cgroup crated
matches with the cgroup path passed by the containers manager.

fixes #1117
fixes #1118
fixes #1021

Signed-off-by: Julio Montes <[email protected]>

virtcontainers/container.go

@@ -9,13 +9,15 @@ package virtcontainers
 import (
 	"context"
 	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"io"
 	"os"
 	"path/filepath"
 	"syscall"
 	"time"
 
+	"github.com/containerd/cgroups"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
 	"github.com/kata-containers/runtime/virtcontainers/types"
 	"github.com/kata-containers/runtime/virtcontainers/utils"
@@ -759,10 +761,8 @@ func (c *Container) create() (err error) {
 	}
 	c.process = *process
 
-	// If this is a sandbox container, store the pid for sandbox
-	ann := c.GetAnnotations()
-	if ann[annotations.ContainerTypeKey] == string(PodSandbox) {
-		c.sandbox.setSandboxPid(c.process.Pid)
+	if err = c.newCgroups(); err != nil {
+		return
 	}
 
 	// Store the container process returned by the agent.
@@ -788,6 +788,10 @@ func (c *Container) delete() error {
 		return err
 	}
 
+	if err := c.deleteCgroups(); err != nil {
+		return err
+	}
+
 	return c.store.Delete()
 }
 
@@ -1055,6 +1059,10 @@ func (c *Container) update(resources specs.LinuxResources) error {
 		return err
 	}
 
+	if err := c.updateCgroups(resources); err != nil {
+		return err
+	}
+
 	return c.sandbox.agent.updateContainer(c.sandbox, *c, resources)
 }
 
@@ -1243,3 +1251,105 @@ func (c *Container) detachDevices() error {
 	}
 	return nil
 }
+
+// creates a new cgroup and return the cgroups path
+func (c *Container) newCgroups() error {
+	ann := c.GetAnnotations()
+
+	config, ok := ann[annotations.ConfigJSONKey]
+	if !ok {
+		return fmt.Errorf("Could not find json config in annotations")
+	}
+
+	var spec specs.Spec
+	if err := json.Unmarshal([]byte(config), &spec); err != nil {
+		return err
+	}
+
+	// https://github.com/kata-containers/runtime/issues/168
+	resources := specs.LinuxResources{
+		CPU: nil,
+	}
+
+	if spec.Linux != nil && spec.Linux.Resources != nil {
+		resources.CPU = validCPUResources(spec.Linux.Resources.CPU)
+	}
+
+	cgroup, err := cgroupsNewFunc(cgroups.V1,
+		cgroups.StaticPath(spec.Linux.CgroupsPath), &resources)
+	if err != nil {
+		return fmt.Errorf("Could not create cgroup for %v: %v", spec.Linux.CgroupsPath, err)
+	}
+
+	c.state.Resources = resources
+	c.state.CgroupPath = spec.Linux.CgroupsPath
+
+	// Add shim into cgroup
+	if c.process.Pid > 0 {
+		if err := cgroup.Add(cgroups.Process{Pid: c.process.Pid}); err != nil {
+			return fmt.Errorf("Could not add PID %d to cgroup %v: %v", c.process.Pid, spec.Linux.CgroupsPath, err)
+		}
+	}
+
+	return nil
+}
+
+func (c *Container) deleteCgroups() error {
+	cgroup, err := cgroupsLoadFunc(cgroups.V1,
+		cgroups.StaticPath(c.state.CgroupPath))
+
+	if err == cgroups.ErrCgroupDeleted {
+		// cgroup already deleted
+		return nil
+	}
+
+	if err != nil {
+		return fmt.Errorf("Could not load container cgroup %v: %v", c.state.CgroupPath, err)
+	}
+
+	// move running process here, that way cgroup can be removed
+	parent, err := parentCgroup(c.state.CgroupPath)
+	if err != nil {
+		// parent cgroup doesn't exist, that means there are no process running
+		// and the container cgroup was removed.
+		c.Logger().WithError(err).Warn("Container cgroup doesn't exist")
+		return nil
+	}
+
+	if err := cgroup.MoveTo(parent); err != nil {
+		// Don't fail, cgroup can be deleted
+		c.Logger().WithError(err).Warn("Could not move container process into parent cgroup")
+	}
+
+	if err := cgroup.Delete(); err != nil {
+		return fmt.Errorf("Could not delete container cgroup %v: %v", c.state.CgroupPath, err)
+	}
+
+	return nil
+}
+
+func (c *Container) updateCgroups(resources specs.LinuxResources) error {
+	cgroup, err := cgroupsLoadFunc(cgroups.V1,
+		cgroups.StaticPath(c.state.CgroupPath))
+	if err != nil {
+		return fmt.Errorf("Could not load cgroup %v: %v", c.state.CgroupPath, err)
+	}
+
+	// Issue: https://github.com/kata-containers/runtime/issues/168
+	r := specs.LinuxResources{
+		CPU: validCPUResources(resources.CPU),
+	}
+
+	// update cgroup
+	if err := cgroup.Update(&r); err != nil {
+		return fmt.Errorf("Could not update cgroup %v: %v", c.state.CgroupPath, err)
+	}
+
+	// store new resources
+	c.state.Resources = r
+	if err := c.store.Store(store.State, c.state); err != nil {
+		return err
+	}
+
+	return nil
+}

virtcontainers/types/sandbox.go

@@ -8,6 +8,8 @@ package types
 import (
 	"fmt"
 	"strings"
+
+	specs "github.com/opencontainers/runtime-spec/specs-go"
 )
 
 // StateString is a string representing a sandbox state.
@@ -44,6 +46,16 @@ type State struct {
 
 	// GuestMemoryBlockSizeMB is the size of memory block of guestos
 	GuestMemoryBlockSizeMB uint32 `json:"guestMemoryBlockSize"`
+
+	// CgroupPath is the cgroup hierarchy where sandbox's processes
+	// including the hypervisor are placed.
+	CgroupPath string `json:"cgroupPath,omitempty"`
+
+	// Resources contains the resources assigned to the container.
+	// When a container is created resources specified in the config json
+	// are used, those resources change when a container is updated but
+	// the config json is not updated.
+	Resources specs.LinuxResources `json:"resources,omitempty"`
 }
 
 // Valid checks that the sandbox state is valid.