cpuset: support setting mems for sandbox

CPUSet cgroup allows for pinning the memory associated with a cpuset to
a given numa node. Similar to cpuset.cpus, we should take cpuset.mems
into account for the sandbox-cgroup that Kata creates.

Fixes: #2970

Signed-off-by: Eric Ernst <[email protected]>

virtcontainers/pkg/cgroups/manager.go

@@ -329,14 +329,15 @@ func (m *Manager) RemoveDevice(device string) error {
 	return fmt.Errorf("device %v not found in the cgroup", device)
 }
 
-func (m *Manager) SetCPUSet(cpuset string) error {
+func (m *Manager) SetCPUSet(cpuset, memset string) error {
 	cgroups, err := m.GetCgroups()
 	if err != nil {
 		return err
 	}
 
 	m.Lock()
 	cgroups.CpusetCpus = cpuset
+	cgroups.CpusetMems = memset
 	m.Unlock()
 
 	return m.Apply()

virtcontainers/sandbox.go

@@ -1959,15 +1959,13 @@ func (s *Sandbox) cgroupsUpdate() error {
 	// in the Kata sandbox cgroup (inherited). Check to see if sandbox cpuset needs to be
 	// updated.
 	if s.config.SandboxCgroupOnly {
-		cpuset, err := s.getSandboxCPUSet()
+		cpuset, memset, err := s.getSandboxCPUSet()
 		if err != nil {
 			return err
 		}
 
-		if cpuset != "" {
-			if err := s.cgroupMgr.SetCPUSet(cpuset); err != nil {
-				return err
-			}
+		if err := s.cgroupMgr.SetCPUSet(cpuset, memset); err != nil {
+			return err
 		}
 
 		return nil
@@ -2264,23 +2262,30 @@ func (s *Sandbox) GetOOMEvent() (string, error) {
 	return s.agent.getOOMEvent()
 }
 
-// getSandboxCPUSet returns the union of each of the sandbox's containers' CPU sets
-// as a string in canonical linux CPU list format
-func (s *Sandbox) getSandboxCPUSet() (string, error) {
+// getSandboxCPUSet returns the union of each of the sandbox's containers' CPU sets'
+// cpus and mems as a string in canonical linux CPU/mems list format
+func (s *Sandbox) getSandboxCPUSet() (string, string, error) {
 	if s.config == nil {
-		return "", nil
+		return "", "", nil
 	}
 
-	result := cpuset.NewCPUSet()
+	cpuResult := cpuset.NewCPUSet()
+	memResult := cpuset.NewCPUSet()
 	for _, ctr := range s.config.Containers {
 		if ctr.Resources.CPU != nil {
-			currSet, err := cpuset.Parse(ctr.Resources.CPU.Cpus)
+			currCpuSet, err := cpuset.Parse(ctr.Resources.CPU.Cpus)
+			if err != nil {
+				return "", "", fmt.Errorf("unable to parse CPUset.cpus for container %s: %v", ctr.ID, err)
+			}
+			cpuResult = cpuResult.Union(currCpuSet)
+
+			currMemSet, err := cpuset.Parse(ctr.Resources.CPU.Mems)
 			if err != nil {
-				return "", fmt.Errorf("unable to parse CPUset for container %s: %v", ctr.ID, err)
+				return "", "", fmt.Errorf("unable to parse CPUset.mems for container %s: %v", ctr.ID, err)
 			}
-			result = result.Union(currSet)
+			memResult = memResult.Union(currMemSet)
 		}
 	}
 
-	return result.String(), nil
+	return cpuResult.String(), memResult.String(), nil
 }

virtcontainers/sandbox_test.go

@@ -1422,24 +1422,25 @@ func TestSandbox_SetupSandboxCgroup(t *testing.T) {
 	}
 }
 
-func getContainerConfigWithCPUSet(cpuset string) ContainerConfig {
+func getContainerConfigWithCPUSet(cpuset, memset string) ContainerConfig {
 	return ContainerConfig{
 		Resources: specs.LinuxResources{
 			CPU: &specs.LinuxCPU{
 				Cpus: cpuset,
+				Mems: memset,
 			},
 		},
 	}
 }
 
-func getSimpleSandbox(cpuset0, cpuset1, cpuset2 string) *Sandbox {
+func getSimpleSandbox(cpusets, memsets [3]string) *Sandbox {
 	sandbox := Sandbox{}
 
 	sandbox.config = &SandboxConfig{
 		Containers: []ContainerConfig{
-			getContainerConfigWithCPUSet(cpuset0),
-			getContainerConfigWithCPUSet(cpuset1),
-			getContainerConfigWithCPUSet(cpuset2),
+			getContainerConfigWithCPUSet(cpusets[0], memsets[0]),
+			getContainerConfigWithCPUSet(cpusets[1], memsets[1]),
+			getContainerConfigWithCPUSet(cpusets[2], memsets[2]),
 		},
 	}
 
@@ -1449,80 +1450,97 @@ func getSimpleSandbox(cpuset0, cpuset1, cpuset2 string) *Sandbox {
 func TestGetSandboxCpuSet(t *testing.T) {
 
 	tests := []struct {
-		name    string
-		cpuset0 string
-		cpuset1 string
-		cpuset2 string
-		result  string
-		wantErr bool
+		name      string
+		cpusets   [3]string
+		memsets   [3]string
+		cpuResult string
+		memResult string
+		wantErr   bool
 	}{
 		{
 			"single, no cpuset",
-			"",
-			"",
+			[3]string{"", "", ""},
+			[3]string{"", "", ""},
 			"",
 			"",
 			false,
 		},
 		{
 			"single cpuset",
+			[3]string{"0", "", ""},
+			[3]string{"", "", ""},
 			"0",
 			"",
-			"",
-			"0",
 			false,
 		},
 		{
 			"two duplicate cpuset",
-			"0",
+			[3]string{"0", "0", ""},
+			[3]string{"", "", ""},
 			"0",
 			"",
-			"0",
 			false,
 		},
 		{
 			"3 cpusets",
-			"0-3",
-			"5-7",
-			"1",
+			[3]string{"0-3", "5-7", "1"},
+			[3]string{"", "", ""},
 			"0-3,5-7",
+			"",
 			false,
 		},
+
 		{
 			"weird, but should be okay",
-			"0-3",
-			"99999",
-			"",
+			[3]string{"0-3", "99999", ""},
+			[3]string{"", "", ""},
 			"0-3,99999",
+			"",
 			false,
 		},
 		{
 			"two, overlapping cpuset",
+			[3]string{"0-3", "1-2", ""},
+			[3]string{"", "", ""},
 			"0-3",
-			"1-2",
 			"",
-			"0-3",
 			false,
 		},
 		{
 			"garbage, should fail",
-			"7 beard-seconds",
-			"Audrey + 7",
-			"Elliott - 17",
+			[3]string{"7 beard-seconds", "Audrey + 7", "Elliott - 17"},
+			[3]string{"", "", ""},
+			"",
 			"",
 			true,
 		},
+		{
+			"cpuset and memset",
+			[3]string{"0-3", "1-2", ""},
+			[3]string{"0", "1", "0-1"},
+			"0-3",
+			"0-1",
+			false,
+		},
+		{
+			"memset",
+			[3]string{"0-3", "1-2", ""},
+			[3]string{"0", "3", ""},
+			"0-3",
+			"0,3",
+			false,
+		},
 	}
 	for _, tt := range tests {
 
 		t.Run(tt.name, func(t *testing.T) {
-			s := getSimpleSandbox(tt.cpuset0, tt.cpuset1, tt.cpuset2)
-			res, err := s.getSandboxCPUSet()
+			s := getSimpleSandbox(tt.cpusets, tt.memsets)
+			res, _, err := s.getSandboxCPUSet()
 			if (err != nil) != tt.wantErr {
 				t.Errorf("getSandboxCPUSet() error = %v, wantErr %v", err, tt.wantErr)
 			}
-			if res != tt.result {
-				t.Errorf("getSandboxCPUSet() result = %s, wanted result %s", res, tt.result)
+			if res != tt.cpuResult {
+				t.Errorf("getSandboxCPUSet() result = %s, wanted result %s", res, tt.cpuResult)
 			}
 		})
 	}