cgroup: ignore cpuset if can not be applied.

The way that kata works is hotplugging cpus to the VM
based in cpu and qouta, cpuset request may not match
with the cpus requested from the host. Apply
only if possible.

Fixes: #446

Signed-off-by: Jose Carlos Venegas Munoz <[email protected]>

cgroup.go

@@ -0,0 +1,65 @@
+//
+// Copyright (c) 2019 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"io/ioutil"
+	"strings"
+
+	"github.com/docker/docker/pkg/parsers"
+	"github.com/sirupsen/logrus"
+)
+
+// set function in variable to overwrite for testing.
+var getCpusetGuest = func() (string, error) {
+	cpusetGuestByte, err := ioutil.ReadFile("/sys/fs/cgroup/cpuset/cpuset.cpus")
+	if err != nil {
+		return "", err
+	}
+
+	return strings.TrimSpace(string(cpusetGuestByte)), nil
+}
+
+// Return the best match for cpuset list in the guest.
+// The runtime caller may apply cpuset for specific CPUs in the host.
+// The CPUs may not exist on the guest as they are hotplugged based
+// on cpu and qouta.
+// This function return a working cpuset to apply on the guest.
+func getAvailableCpusetList(cpusetReq string) (string, error) {
+
+	cpusetGuest, err := getCpusetGuest()
+	if err != nil {
+		return "", err
+	}
+
+	cpusetListReq, err := parsers.ParseUintList(cpusetReq)
+	if err != nil {
+		return "", err
+	}
+
+	cpusetGuestList, err := parsers.ParseUintList(cpusetGuest)
+	if err != nil {
+		return "", err
+	}
+
+	for k := range cpusetListReq {
+		if !cpusetGuestList[k] {
+			agentLog.WithFields(logrus.Fields{
+				"cpuset":     cpusetReq,
+				"cpu":        k,
+				"guest-cpus": cpusetGuest,
+			}).Warnf("cpu is not in guest cpu list, using guest cpus")
+			return cpusetGuest, nil
+		}
+	}
+
+	// All the cpus are valid keep the same cpuset string
+	agentLog.WithFields(logrus.Fields{
+		"cpuset": cpusetReq,
+	}).Debugf("the requested cpuset is valid, using it")
+	return cpusetReq, nil
+}

cgroup_test.go

@@ -0,0 +1,45 @@
+//  Copyright (c) 2019 Intel Corporation
+//
+//  SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetAvailableCpusetList(t *testing.T) {
+	fakeGuestCpuset := "0-3"
+	getCpusetGuest = func() (string, error) {
+		return fakeGuestCpuset, nil
+	}
+
+	type testCase struct {
+		input          string
+		expectedOutput string
+	}
+
+	cases := []testCase{
+		{"0", "0"},
+		{"0,1", "0,1"},
+		{"0,1,2", "0,1,2"},
+		{"0,1,2,3", "0,1,2,3"},
+		{"0,1,2,3,4", fakeGuestCpuset},
+		{"0-3", "0-3"},
+		{"0-3,4", fakeGuestCpuset},
+		{"0-4", fakeGuestCpuset},
+		{"1", "1"},
+		{"1,3", "1,3"},
+		{"2-3", "2-3"},
+		{"2-4", fakeGuestCpuset},
+	}
+
+	for _, c := range cases {
+		out, err := getAvailableCpusetList(c.input)
+		assert.Nil(t, err, "Failed to calculate : %v", err)
+		assert.Equal(t, out, c.expectedOutput)
+	}
+}

documentation/features/cpuset.md

@@ -0,0 +1,52 @@
+# Cpuset cgroup.
+
+From Kernel documentation:
+
+_" Cpusets provide a mechanism for assigning a set of CPUs and Memory Nodes to
+a set of tasks."_
+
+The Kata agent brings compatibility to the cgroup cpuset CPU on the guest side.
+
+The cpuset CPU cgroup will be applied on two events:
+
+- containers creation
+
+- container update
+
+When the runtime requests to apply cpuset cgroup to the agent, the amount of
+vcpus available might not be the same to the required vcpus in the request.
+
+This is because the request from the agent client (i.e. the Kata runtime)
+passes cpusets that are requested to be placed on the host. This isolates the
+container workload on some specific host CPUs. The runtime passes the requested
+cpuset to the agent, which tries to apply the cgroup cpuset on the guest.
+
+The runtime only calculates and hot-plugs the CPUSs based on the container
+period and quota. This is why the VM will not have the same amount of CPUs as
+the host.
+
+Example:
+
+docker run -ti --cpus 2 --cpuset 0,1 busybox
+
+This should result with the container limited to the time of 2 CPUs, but is
+only allowed to be scheduled on CPUs 0 and 1.
+
+The following is an example of a similar case with a valid traditional container:
+
+docker run -ti --cpus 2 --cpuset 2,3,4 busybox
+
+Here, the container is limited to 2 CPUs and can be scheduled on CPU 2, 3, and
+4.
+
+The Kata runtime only hotplugs 2 CPUs, making it impossible to request that the
+guest kernel schedules the workload on vcpu 3 and 4.
+
+## cpuset best effort application.
+
+The Kata agent evaluates the request to see if it is possible to apply the
+cpuset request onto the guest.
+
+- If the CPUSs requested are not available in the guest, the request is ignored.
+- If the CPUs requested are available, the request is applied by the agent.
+

grpc.go

@@ -179,7 +179,7 @@ func updateContainerCpuset(cgroupPath string, newCpuset string, cookies cookie)
 		// Don't use c.container.Set because of it will modify container's config.
 		// c.container.Set MUST BE used only on update.
 		cpusetCpusPath := filepath.Join(cpusetPath, "cpuset.cpus")
-		agentLog.WithField("path", cpusetPath).Debug("updating cpuset cgroup")
+
 		if err := ioutil.WriteFile(cpusetCpusPath, []byte(newCpuset), cpusetMode); err != nil {
 			return fmt.Errorf("Could not update cpuset cgroup '%s': %v", newCpuset, err)
 		}
@@ -625,6 +625,15 @@ func (a *agentGRPC) CreateContainer(ctx context.Context, req *pb.CreateContainer
 		return emptyResp, err
 	}
 
+	if ociSpec.Linux.Resources.CPU != nil && ociSpec.Linux.Resources.CPU.Cpus != "" {
+		availableCpuset, err := getAvailableCpusetList(ociSpec.Linux.Resources.CPU.Cpus)
+		if err != nil {
+			return emptyResp, err
+		}
+
+		ociSpec.Linux.Resources.CPU.Cpus = availableCpuset
+	}
+
 	if a.sandbox.guestHooksPresent {
 		// Add any custom OCI hooks to the spec
 		a.sandbox.addGuestHooks(ociSpec)
@@ -1017,6 +1026,11 @@ func (a *agentGRPC) UpdateContainer(ctx context.Context, req *pb.UpdateContainer
 
 	// cpuset is a special case where container's cpuset cgroup MUST BE updated
 	if resources.CpusetCpus != "" {
+		resources.CpusetCpus, err = getAvailableCpusetList(resources.CpusetCpus)
+		if err != nil {
+			return emptyResp, err
+		}
+
 		cookies := make(cookie)
 		if err = updateContainerCpuset(contConfig.Cgroups.Path, resources.CpusetCpus, cookies); err != nil {
 			agentLog.WithError(err).Warn("Could not update container cpuset cgroup")