From 0e0ef63328e77b48fd768f30855f7f8ff682cba1 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Fri, 18 Sep 2020 14:55:58 -0700 Subject: [PATCH] cpuset: support setting mems for sandbox CPUSet cgroup allows for pinning the memory associated with a cpuset to a given numa node. Similar to cpuset.cpus, we should take cpuset.mems into account for the sandbox-cgroup that Kata creates. Fixes: #2970 Signed-off-by: Eric Ernst --- virtcontainers/pkg/cgroups/manager.go | 3 +- virtcontainers/sandbox.go | 33 ++++++----- virtcontainers/sandbox_test.go | 82 ++++++++++++++++----------- 3 files changed, 71 insertions(+), 47 deletions(-) diff --git a/virtcontainers/pkg/cgroups/manager.go b/virtcontainers/pkg/cgroups/manager.go index 3378ad627a..34024a4386 100644 --- a/virtcontainers/pkg/cgroups/manager.go +++ b/virtcontainers/pkg/cgroups/manager.go @@ -329,7 +329,7 @@ func (m *Manager) RemoveDevice(device string) error { return fmt.Errorf("device %v not found in the cgroup", device) } -func (m *Manager) SetCPUSet(cpuset string) error { +func (m *Manager) SetCPUSet(cpuset, memset string) error { cgroups, err := m.GetCgroups() if err != nil { return err @@ -337,6 +337,7 @@ func (m *Manager) SetCPUSet(cpuset string) error { m.Lock() cgroups.CpusetCpus = cpuset + cgroups.CpusetMems = memset m.Unlock() return m.Apply() diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index 6f8cb41c2d..c7ae0f60bc 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -1959,15 +1959,13 @@ func (s *Sandbox) cgroupsUpdate() error { // in the Kata sandbox cgroup (inherited). Check to see if sandbox cpuset needs to be // updated. if s.config.SandboxCgroupOnly { - cpuset, err := s.getSandboxCPUSet() + cpuset, memset, err := s.getSandboxCPUSet() if err != nil { return err } - if cpuset != "" { - if err := s.cgroupMgr.SetCPUSet(cpuset); err != nil { - return err - } + if err := s.cgroupMgr.SetCPUSet(cpuset, memset); err != nil { + return err } return nil @@ -2264,23 +2262,30 @@ func (s *Sandbox) GetOOMEvent() (string, error) { return s.agent.getOOMEvent() } -// getSandboxCPUSet returns the union of each of the sandbox's containers' CPU sets -// as a string in canonical linux CPU list format -func (s *Sandbox) getSandboxCPUSet() (string, error) { +// getSandboxCPUSet returns the union of each of the sandbox's containers' CPU sets' +// cpus and mems as a string in canonical linux CPU/mems list format +func (s *Sandbox) getSandboxCPUSet() (string, string, error) { if s.config == nil { - return "", nil + return "", "", nil } - result := cpuset.NewCPUSet() + cpuResult := cpuset.NewCPUSet() + memResult := cpuset.NewCPUSet() for _, ctr := range s.config.Containers { if ctr.Resources.CPU != nil { - currSet, err := cpuset.Parse(ctr.Resources.CPU.Cpus) + currCpuSet, err := cpuset.Parse(ctr.Resources.CPU.Cpus) + if err != nil { + return "", "", fmt.Errorf("unable to parse CPUset.cpus for container %s: %v", ctr.ID, err) + } + cpuResult = cpuResult.Union(currCpuSet) + + currMemSet, err := cpuset.Parse(ctr.Resources.CPU.Mems) if err != nil { - return "", fmt.Errorf("unable to parse CPUset for container %s: %v", ctr.ID, err) + return "", "", fmt.Errorf("unable to parse CPUset.mems for container %s: %v", ctr.ID, err) } - result = result.Union(currSet) + memResult = memResult.Union(currMemSet) } } - return result.String(), nil + return cpuResult.String(), memResult.String(), nil } diff --git a/virtcontainers/sandbox_test.go b/virtcontainers/sandbox_test.go index 7487391438..bfc340ccf7 100644 --- a/virtcontainers/sandbox_test.go +++ b/virtcontainers/sandbox_test.go @@ -1422,24 +1422,25 @@ func TestSandbox_SetupSandboxCgroup(t *testing.T) { } } -func getContainerConfigWithCPUSet(cpuset string) ContainerConfig { +func getContainerConfigWithCPUSet(cpuset, memset string) ContainerConfig { return ContainerConfig{ Resources: specs.LinuxResources{ CPU: &specs.LinuxCPU{ Cpus: cpuset, + Mems: memset, }, }, } } -func getSimpleSandbox(cpuset0, cpuset1, cpuset2 string) *Sandbox { +func getSimpleSandbox(cpusets, memsets [3]string) *Sandbox { sandbox := Sandbox{} sandbox.config = &SandboxConfig{ Containers: []ContainerConfig{ - getContainerConfigWithCPUSet(cpuset0), - getContainerConfigWithCPUSet(cpuset1), - getContainerConfigWithCPUSet(cpuset2), + getContainerConfigWithCPUSet(cpusets[0], memsets[0]), + getContainerConfigWithCPUSet(cpusets[1], memsets[1]), + getContainerConfigWithCPUSet(cpusets[2], memsets[2]), }, } @@ -1449,80 +1450,97 @@ func getSimpleSandbox(cpuset0, cpuset1, cpuset2 string) *Sandbox { func TestGetSandboxCpuSet(t *testing.T) { tests := []struct { - name string - cpuset0 string - cpuset1 string - cpuset2 string - result string - wantErr bool + name string + cpusets [3]string + memsets [3]string + cpuResult string + memResult string + wantErr bool }{ { "single, no cpuset", - "", - "", + [3]string{"", "", ""}, + [3]string{"", "", ""}, "", "", false, }, { "single cpuset", + [3]string{"0", "", ""}, + [3]string{"", "", ""}, "0", "", - "", - "0", false, }, { "two duplicate cpuset", - "0", + [3]string{"0", "0", ""}, + [3]string{"", "", ""}, "0", "", - "0", false, }, { "3 cpusets", - "0-3", - "5-7", - "1", + [3]string{"0-3", "5-7", "1"}, + [3]string{"", "", ""}, "0-3,5-7", + "", false, }, + { "weird, but should be okay", - "0-3", - "99999", - "", + [3]string{"0-3", "99999", ""}, + [3]string{"", "", ""}, "0-3,99999", + "", false, }, { "two, overlapping cpuset", + [3]string{"0-3", "1-2", ""}, + [3]string{"", "", ""}, "0-3", - "1-2", "", - "0-3", false, }, { "garbage, should fail", - "7 beard-seconds", - "Audrey + 7", - "Elliott - 17", + [3]string{"7 beard-seconds", "Audrey + 7", "Elliott - 17"}, + [3]string{"", "", ""}, + "", "", true, }, + { + "cpuset and memset", + [3]string{"0-3", "1-2", ""}, + [3]string{"0", "1", "0-1"}, + "0-3", + "0-1", + false, + }, + { + "memset", + [3]string{"0-3", "1-2", ""}, + [3]string{"0", "3", ""}, + "0-3", + "0,3", + false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := getSimpleSandbox(tt.cpuset0, tt.cpuset1, tt.cpuset2) - res, err := s.getSandboxCPUSet() + s := getSimpleSandbox(tt.cpusets, tt.memsets) + res, _, err := s.getSandboxCPUSet() if (err != nil) != tt.wantErr { t.Errorf("getSandboxCPUSet() error = %v, wantErr %v", err, tt.wantErr) } - if res != tt.result { - t.Errorf("getSandboxCPUSet() result = %s, wanted result %s", res, tt.result) + if res != tt.cpuResult { + t.Errorf("getSandboxCPUSet() result = %s, wanted result %s", res, tt.cpuResult) } }) }